Total coverage: 452241 (23%)of 1987592 



















































   15 

   20 

   15 





   21 















   25 










   49 














   39 




   36 











   40 




   40 
   56 










   21 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BTREE_CACHE_H
#define _BCACHEFS_BTREE_CACHE_H

#include "bcachefs.h"
#include "btree_types.h"
#include "bkey_methods.h"

extern const char * const bch2_btree_node_flags[];

struct btree_iter;

void bch2_recalc_btree_reserve(struct bch_fs *);

void bch2_btree_node_to_freelist(struct bch_fs *, struct btree *);

void __bch2_btree_node_hash_remove(struct btree_cache *, struct btree *);
void bch2_btree_node_hash_remove(struct btree_cache *, struct btree *);

int __bch2_btree_node_hash_insert(struct btree_cache *, struct btree *);
int bch2_btree_node_hash_insert(struct btree_cache *, struct btree *,
                                unsigned, enum btree_id);

void bch2_node_pin(struct bch_fs *, struct btree *);
void bch2_btree_cache_unpin(struct bch_fs *);

void bch2_btree_node_update_key_early(struct btree_trans *, enum btree_id, unsigned,
                                      struct bkey_s_c, struct bkey_i *);

void bch2_btree_cache_cannibalize_unlock(struct btree_trans *);
int bch2_btree_cache_cannibalize_lock(struct btree_trans *, struct closure *);

struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *);
struct btree *bch2_btree_node_mem_alloc(struct btree_trans *, bool);

struct btree *bch2_btree_node_get(struct btree_trans *, struct btree_path *,
                                  const struct bkey_i *, unsigned,
                                  enum six_lock_type, unsigned long);

struct btree *bch2_btree_node_get_noiter(struct btree_trans *, const struct bkey_i *,
                                         enum btree_id, unsigned, bool);

int bch2_btree_node_prefetch(struct btree_trans *, struct btree_path *,
                             const struct bkey_i *, enum btree_id, unsigned);

void bch2_btree_node_evict(struct btree_trans *, const struct bkey_i *);

void bch2_fs_btree_cache_exit(struct bch_fs *);
int bch2_fs_btree_cache_init(struct bch_fs *);
void bch2_fs_btree_cache_init_early(struct btree_cache *);

static inline u64 btree_ptr_hash_val(const struct bkey_i *k)
{
        switch (k->k.type) {
        case KEY_TYPE_btree_ptr:
                return *((u64 *) bkey_i_to_btree_ptr_c(k)->v.start);
        case KEY_TYPE_btree_ptr_v2:
                /*
                 * The cast/deref is only necessary to avoid sparse endianness
                 * warnings:
                 */
                return *((u64 *) &bkey_i_to_btree_ptr_v2_c(k)->v.seq);
        default:
                return 0;
        }
}

static inline struct btree *btree_node_mem_ptr(const struct bkey_i *k)
{
        return k->k.type == KEY_TYPE_btree_ptr_v2
                ? (void *)(unsigned long)bkey_i_to_btree_ptr_v2_c(k)->v.mem_ptr
                : NULL;
}

/* is btree node in hash table? */
static inline bool btree_node_hashed(struct btree *b)
{
        return b->hash_val != 0;
}

#define for_each_cached_btree(_b, _c, _tbl, _iter, _pos)                \
        for ((_tbl) = rht_dereference_rcu((_c)->btree_cache.table.tbl,        \
                                          &(_c)->btree_cache.table),        \
             _iter = 0;        _iter < (_tbl)->size; _iter++)                        \
                rht_for_each_entry_rcu((_b), (_pos), _tbl, _iter, hash)

static inline size_t btree_buf_bytes(const struct btree *b)
{
        return 1UL << b->byte_order;
}

static inline size_t btree_buf_max_u64s(const struct btree *b)
{
        return (btree_buf_bytes(b) - sizeof(struct btree_node)) / sizeof(u64);
}

static inline size_t btree_max_u64s(const struct bch_fs *c)
{
        return (c->opts.btree_node_size - sizeof(struct btree_node)) / sizeof(u64);
}

static inline size_t btree_sectors(const struct bch_fs *c)
{
        return c->opts.btree_node_size >> SECTOR_SHIFT;
}

static inline unsigned btree_blocks(const struct bch_fs *c)
{
        return btree_sectors(c) >> c->block_bits;
}

#define BTREE_SPLIT_THRESHOLD(c)                (btree_max_u64s(c) * 2 / 3)

#define BTREE_FOREGROUND_MERGE_THRESHOLD(c)        (btree_max_u64s(c) * 1 / 3)
#define BTREE_FOREGROUND_MERGE_HYSTERESIS(c)                        \
        (BTREE_FOREGROUND_MERGE_THRESHOLD(c) +                        \
         (BTREE_FOREGROUND_MERGE_THRESHOLD(c) >> 2))

static inline unsigned btree_id_nr_alive(struct bch_fs *c)
{
        return BTREE_ID_NR + c->btree_roots_extra.nr;
}

static inline struct btree_root *bch2_btree_id_root(struct bch_fs *c, unsigned id)
{
        if (likely(id < BTREE_ID_NR)) {
                return &c->btree_roots_known[id];
        } else {
                unsigned idx = id - BTREE_ID_NR;

                EBUG_ON(idx >= c->btree_roots_extra.nr);
                return &c->btree_roots_extra.data[idx];
        }
}

static inline struct btree *btree_node_root(struct bch_fs *c, struct btree *b)
{
        return bch2_btree_id_root(c, b->c.btree_id)->b;
}

const char *bch2_btree_id_str(enum btree_id);
void bch2_btree_id_to_text(struct printbuf *, enum btree_id);

void bch2_btree_pos_to_text(struct printbuf *, struct bch_fs *, const struct btree *);
void bch2_btree_node_to_text(struct printbuf *, struct bch_fs *, const struct btree *);
void bch2_btree_cache_to_text(struct printbuf *, const struct btree_cache *);

#endif /* _BCACHEFS_BTREE_CACHE_H */


























































    7 
    7 




    6 








    7 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * DES & Triple DES EDE key verification helpers
 */

#ifndef __CRYPTO_INTERNAL_DES_H
#define __CRYPTO_INTERNAL_DES_H

#include <linux/crypto.h>
#include <linux/fips.h>
#include <crypto/des.h>
#include <crypto/aead.h>
#include <crypto/skcipher.h>

/**
 * crypto_des_verify_key - Check whether a DES key is weak
 * @tfm: the crypto algo
 * @key: the key buffer
 *
 * Returns -EINVAL if the key is weak and the crypto TFM does not permit weak
 * keys. Otherwise, 0 is returned.
 *
 * It is the job of the caller to ensure that the size of the key equals
 * DES_KEY_SIZE.
 */
static inline int crypto_des_verify_key(struct crypto_tfm *tfm, const u8 *key)
{
        struct des_ctx tmp;
        int err;

        err = des_expand_key(&tmp, key, DES_KEY_SIZE);
        if (err == -ENOKEY) {
                if (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)
                        err = -EINVAL;
                else
                        err = 0;
        }
        memzero_explicit(&tmp, sizeof(tmp));
        return err;
}

/*
 * RFC2451:
 *
 *   For DES-EDE3, there is no known need to reject weak or
 *   complementation keys.  Any weakness is obviated by the use of
 *   multiple keys.
 *
 *   However, if the first two or last two independent 64-bit keys are
 *   equal (k1 == k2 or k2 == k3), then the DES3 operation is simply the
 *   same as DES.  Implementers MUST reject keys that exhibit this
 *   property.
 *
 */
static inline int des3_ede_verify_key(const u8 *key, unsigned int key_len,
                                      bool check_weak)
{
        int ret = fips_enabled ? -EINVAL : -ENOKEY;
        u32 K[6];

        memcpy(K, key, DES3_EDE_KEY_SIZE);

        if ((!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
             !((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
            (fips_enabled || check_weak))
                goto bad;

        if ((!((K[0] ^ K[4]) | (K[1] ^ K[5]))) && fips_enabled)
                goto bad;

        ret = 0;
bad:
        memzero_explicit(K, DES3_EDE_KEY_SIZE);

        return ret;
}

/**
 * crypto_des3_ede_verify_key - Check whether a DES3-EDE key is weak
 * @tfm: the crypto algo
 * @key: the key buffer
 *
 * Returns -EINVAL if the key is weak and the crypto TFM does not permit weak
 * keys or when running in FIPS mode. Otherwise, 0 is returned. Note that some
 * keys are rejected in FIPS mode even if weak keys are permitted by the TFM
 * flags.
 *
 * It is the job of the caller to ensure that the size of the key equals
 * DES3_EDE_KEY_SIZE.
 */
static inline int crypto_des3_ede_verify_key(struct crypto_tfm *tfm,
                                             const u8 *key)
{
        return des3_ede_verify_key(key, DES3_EDE_KEY_SIZE,
                                   crypto_tfm_get_flags(tfm) &
                                   CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);
}

static inline int verify_skcipher_des_key(struct crypto_skcipher *tfm,
                                          const u8 *key)
{
        return crypto_des_verify_key(crypto_skcipher_tfm(tfm), key);
}

static inline int verify_skcipher_des3_key(struct crypto_skcipher *tfm,
                                           const u8 *key)
{
        return crypto_des3_ede_verify_key(crypto_skcipher_tfm(tfm), key);
}

static inline int verify_aead_des_key(struct crypto_aead *tfm, const u8 *key,
                                      int keylen)
{
        if (keylen != DES_KEY_SIZE)
                return -EINVAL;
        return crypto_des_verify_key(crypto_aead_tfm(tfm), key);
}

static inline int verify_aead_des3_key(struct crypto_aead *tfm, const u8 *key,
                                       int keylen)
{
        if (keylen != DES3_EDE_KEY_SIZE)
                return -EINVAL;
        return crypto_des3_ede_verify_key(crypto_aead_tfm(tfm), key);
}

#endif /* __CRYPTO_INTERNAL_DES_H */



















   48 




















   42 
   42 
















   48 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
// SPDX-License-Identifier: GPL-2.0
/*
 *  fs/partitions/karma.c
 *  Rio Karma partition info.
 *
 *  Copyright (C) 2006 Bob Copeland (me@bobcopeland.com)
 *  based on osf.c
 */

#include "check.h"
#include <linux/compiler.h>

#define KARMA_LABEL_MAGIC                0xAB56

int karma_partition(struct parsed_partitions *state)
{
        int i;
        int slot = 1;
        Sector sect;
        unsigned char *data;
        struct disklabel {
                u8 d_reserved[270];
                struct d_partition {
                        __le32 p_res;
                        u8 p_fstype;
                        u8 p_res2[3];
                        __le32 p_offset;
                        __le32 p_size;
                } d_partitions[2];
                u8 d_blank[208];
                __le16 d_magic;
        } __packed *label;
        struct d_partition *p;

        data = read_part_sector(state, 0, &sect);
        if (!data)
                return -1;

        label = (struct disklabel *)data;
        if (le16_to_cpu(label->d_magic) != KARMA_LABEL_MAGIC) {
                put_dev_sector(sect);
                return 0;
        }

        p = label->d_partitions;
        for (i = 0 ; i < 2; i++, p++) {
                if (slot == state->limit)
                        break;

                if (p->p_fstype == 0x4d && le32_to_cpu(p->p_size)) {
                        put_partition(state, slot, le32_to_cpu(p->p_offset),
                                le32_to_cpu(p->p_size));
                }
                slot++;
        }
        strlcat(state->pp_buf, "\n", PAGE_SIZE);
        put_dev_sector(sect);
        return 1;
}
























    4 





    1 













   36 
























































    3 






    4 


    4 












    4 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
/*
 * Copyright (c) Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under both the BSD-style license (found in the
 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
 * in the COPYING file in the root directory of this source tree).
 * You may select, at your option, one of the above-listed licenses.
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/zstd.h>

#include "common/zstd_deps.h"

/* Common symbols. zstd_compress must depend on zstd_decompress. */

unsigned int zstd_is_error(size_t code)
{
        return ZSTD_isError(code);
}
EXPORT_SYMBOL(zstd_is_error);

zstd_error_code zstd_get_error_code(size_t code)
{
        return ZSTD_getErrorCode(code);
}
EXPORT_SYMBOL(zstd_get_error_code);

const char *zstd_get_error_name(size_t code)
{
        return ZSTD_getErrorName(code);
}
EXPORT_SYMBOL(zstd_get_error_name);

/* Decompression symbols. */

size_t zstd_dctx_workspace_bound(void)
{
        return ZSTD_estimateDCtxSize();
}
EXPORT_SYMBOL(zstd_dctx_workspace_bound);

zstd_dctx *zstd_create_dctx_advanced(zstd_custom_mem custom_mem)
{
        return ZSTD_createDCtx_advanced(custom_mem);
}
EXPORT_SYMBOL(zstd_create_dctx_advanced);

size_t zstd_free_dctx(zstd_dctx *dctx)
{
        return ZSTD_freeDCtx(dctx);
}
EXPORT_SYMBOL(zstd_free_dctx);

zstd_ddict *zstd_create_ddict_byreference(const void *dict, size_t dict_size,
                                          zstd_custom_mem custom_mem)
{
        return ZSTD_createDDict_advanced(dict, dict_size, ZSTD_dlm_byRef,
                                         ZSTD_dct_auto, custom_mem);

}
EXPORT_SYMBOL(zstd_create_ddict_byreference);

size_t zstd_free_ddict(zstd_ddict *ddict)
{
        return ZSTD_freeDDict(ddict);
}
EXPORT_SYMBOL(zstd_free_ddict);

zstd_dctx *zstd_init_dctx(void *workspace, size_t workspace_size)
{
        if (workspace == NULL)
                return NULL;
        return ZSTD_initStaticDCtx(workspace, workspace_size);
}
EXPORT_SYMBOL(zstd_init_dctx);

size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
        const void *src, size_t src_size)
{
        return ZSTD_decompressDCtx(dctx, dst, dst_capacity, src, src_size);
}
EXPORT_SYMBOL(zstd_decompress_dctx);

size_t zstd_decompress_using_ddict(zstd_dctx *dctx,
        void *dst, size_t dst_capacity, const void* src, size_t src_size,
        const zstd_ddict* ddict)
{
        return ZSTD_decompress_usingDDict(dctx, dst, dst_capacity, src,
                                          src_size, ddict);
}
EXPORT_SYMBOL(zstd_decompress_using_ddict);

size_t zstd_dstream_workspace_bound(size_t max_window_size)
{
        return ZSTD_estimateDStreamSize(max_window_size);
}
EXPORT_SYMBOL(zstd_dstream_workspace_bound);

zstd_dstream *zstd_init_dstream(size_t max_window_size, void *workspace,
        size_t workspace_size)
{
        if (workspace == NULL)
                return NULL;
        (void)max_window_size;
        return ZSTD_initStaticDStream(workspace, workspace_size);
}
EXPORT_SYMBOL(zstd_init_dstream);

size_t zstd_reset_dstream(zstd_dstream *dstream)
{
        return ZSTD_resetDStream(dstream);
}
EXPORT_SYMBOL(zstd_reset_dstream);

size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
        zstd_in_buffer *input)
{
        return ZSTD_decompressStream(dstream, output, input);
}
EXPORT_SYMBOL(zstd_decompress_stream);

size_t zstd_find_frame_compressed_size(const void *src, size_t src_size)
{
        return ZSTD_findFrameCompressedSize(src, src_size);
}
EXPORT_SYMBOL(zstd_find_frame_compressed_size);

size_t zstd_get_frame_header(zstd_frame_header *header, const void *src,
        size_t src_size)
{
        return ZSTD_getFrameHeader(header, src, src_size);
}
EXPORT_SYMBOL(zstd_get_frame_header);

MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("Zstd Decompressor");



























































   31 







































   56 




   26 




   33 

   12 



   26 
   25 
   25 
   26 



   14 

   15 

   53 

   57 












   24 






   25 

   25 
   25 






    3 
   22 

   25 
   25 

    3 










   55 







   56 
   57 
   54 
   54 
   57 

   55 









   55 




   54 
   55 

   52 
















   54 
   25 
   52 










   34 

   34 



























   33 


   34 






   32 


   33 
   34 



















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
/*
 *                INETPEER - A storage for permanent information about peers
 *
 *  This source is covered by the GNU GPL, the same as all kernel sources.
 *
 *  Authors:        Andrey V. Savochkin <saw@msu.ru>
 */

#include <linux/cache.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/random.h>
#include <linux/timer.h>
#include <linux/time.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/net.h>
#include <linux/workqueue.h>
#include <net/ip.h>
#include <net/inetpeer.h>
#include <net/secure_seq.h>

/*
 *  Theory of operations.
 *  We keep one entry for each peer IP address.  The nodes contains long-living
 *  information about the peer which doesn't depend on routes.
 *
 *  Nodes are removed only when reference counter goes to 0.
 *  When it's happened the node may be removed when a sufficient amount of
 *  time has been passed since its last use.  The less-recently-used entry can
 *  also be removed if the pool is overloaded i.e. if the total amount of
 *  entries is greater-or-equal than the threshold.
 *
 *  Node pool is organised as an RB tree.
 *  Such an implementation has been chosen not just for fun.  It's a way to
 *  prevent easy and efficient DoS attacks by creating hash collisions.  A huge
 *  amount of long living nodes in a single hash slot would significantly delay
 *  lookups performed with disabled BHs.
 *
 *  Serialisation issues.
 *  1.  Nodes may appear in the tree only with the pool lock held.
 *  2.  Nodes may disappear from the tree only with the pool lock held
 *      AND reference count being 0.
 *  3.  Global variable peer_total is modified under the pool lock.
 *  4.  struct inet_peer fields modification:
 *                rb_node: pool lock
 *                refcnt: atomically against modifications on other CPU;
 *                   usually under some other lock to prevent node disappearing
 *                daddr: unchangeable
 */

static struct kmem_cache *peer_cachep __ro_after_init;

void inet_peer_base_init(struct inet_peer_base *bp)
{
        bp->rb_root = RB_ROOT;
        seqlock_init(&bp->lock);
        bp->total = 0;
}
EXPORT_SYMBOL_GPL(inet_peer_base_init);

#define PEER_MAX_GC 32

/* Exported for sysctl_net_ipv4.  */
int inet_peer_threshold __read_mostly;        /* start to throw entries more
                                         * aggressively at this stage */
int inet_peer_minttl __read_mostly = 120 * HZ;        /* TTL under high load: 120 sec */
int inet_peer_maxttl __read_mostly = 10 * 60 * HZ;        /* usual time to live: 10 min */

/* Called from ip_output.c:ip_init  */
void __init inet_initpeers(void)
{
        u64 nr_entries;

         /* 1% of physical memory */
        nr_entries = div64_ul((u64)totalram_pages() << PAGE_SHIFT,
                              100 * L1_CACHE_ALIGN(sizeof(struct inet_peer)));

        inet_peer_threshold = clamp_val(nr_entries, 4096, 65536 + 128);

        peer_cachep = KMEM_CACHE(inet_peer, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
}

/* Called with rcu_read_lock() or base->lock held */
static struct inet_peer *lookup(const struct inetpeer_addr *daddr,
                                struct inet_peer_base *base,
                                unsigned int seq,
                                struct inet_peer *gc_stack[],
                                unsigned int *gc_cnt,
                                struct rb_node **parent_p,
                                struct rb_node ***pp_p)
{
        struct rb_node **pp, *parent, *next;
        struct inet_peer *p;

        pp = &base->rb_root.rb_node;
        parent = NULL;
        while (1) {
                int cmp;

                next = rcu_dereference_raw(*pp);
                if (!next)
                        break;
                parent = next;
                p = rb_entry(parent, struct inet_peer, rb_node);
                cmp = inetpeer_addr_cmp(daddr, &p->daddr);
                if (cmp == 0) {
                        if (!refcount_inc_not_zero(&p->refcnt))
                                break;
                        return p;
                }
                if (gc_stack) {
                        if (*gc_cnt < PEER_MAX_GC)
                                gc_stack[(*gc_cnt)++] = p;
                } else if (unlikely(read_seqretry(&base->lock, seq))) {
                        break;
                }
                if (cmp == -1)
                        pp = &next->rb_left;
                else
                        pp = &next->rb_right;
        }
        *parent_p = parent;
        *pp_p = pp;
        return NULL;
}

/* perform garbage collect on all items stacked during a lookup */
static void inet_peer_gc(struct inet_peer_base *base,
                         struct inet_peer *gc_stack[],
                         unsigned int gc_cnt)
{
        int peer_threshold, peer_maxttl, peer_minttl;
        struct inet_peer *p;
        __u32 delta, ttl;
        int i;

        peer_threshold = READ_ONCE(inet_peer_threshold);
        peer_maxttl = READ_ONCE(inet_peer_maxttl);
        peer_minttl = READ_ONCE(inet_peer_minttl);

        if (base->total >= peer_threshold)
                ttl = 0; /* be aggressive */
        else
                ttl = peer_maxttl - (peer_maxttl - peer_minttl) / HZ *
                        base->total / peer_threshold * HZ;
        for (i = 0; i < gc_cnt; i++) {
                p = gc_stack[i];

                /* The READ_ONCE() pairs with the WRITE_ONCE()
                 * in inet_putpeer()
                 */
                delta = (__u32)jiffies - READ_ONCE(p->dtime);

                if (delta < ttl || !refcount_dec_if_one(&p->refcnt))
                        gc_stack[i] = NULL;
        }
        for (i = 0; i < gc_cnt; i++) {
                p = gc_stack[i];
                if (p) {
                        rb_erase(&p->rb_node, &base->rb_root);
                        base->total--;
                        kfree_rcu(p, rcu);
                }
        }
}

struct inet_peer *inet_getpeer(struct inet_peer_base *base,
                               const struct inetpeer_addr *daddr,
                               int create)
{
        struct inet_peer *p, *gc_stack[PEER_MAX_GC];
        struct rb_node **pp, *parent;
        unsigned int gc_cnt, seq;
        int invalidated;

        /* Attempt a lockless lookup first.
         * Because of a concurrent writer, we might not find an existing entry.
         */
        rcu_read_lock();
        seq = read_seqbegin(&base->lock);
        p = lookup(daddr, base, seq, NULL, &gc_cnt, &parent, &pp);
        invalidated = read_seqretry(&base->lock, seq);
        rcu_read_unlock();

        if (p)
                return p;

        /* If no writer did a change during our lookup, we can return early. */
        if (!create && !invalidated)
                return NULL;

        /* retry an exact lookup, taking the lock before.
         * At least, nodes should be hot in our cache.
         */
        parent = NULL;
        write_seqlock_bh(&base->lock);

        gc_cnt = 0;
        p = lookup(daddr, base, seq, gc_stack, &gc_cnt, &parent, &pp);
        if (!p && create) {
                p = kmem_cache_alloc(peer_cachep, GFP_ATOMIC);
                if (p) {
                        p->daddr = *daddr;
                        p->dtime = (__u32)jiffies;
                        refcount_set(&p->refcnt, 2);
                        atomic_set(&p->rid, 0);
                        p->metrics[RTAX_LOCK-1] = INETPEER_METRICS_NEW;
                        p->rate_tokens = 0;
                        p->n_redirects = 0;
                        /* 60*HZ is arbitrary, but chosen enough high so that the first
                         * calculation of tokens is at its maximum.
                         */
                        p->rate_last = jiffies - 60*HZ;

                        rb_link_node(&p->rb_node, parent, pp);
                        rb_insert_color(&p->rb_node, &base->rb_root);
                        base->total++;
                }
        }
        if (gc_cnt)
                inet_peer_gc(base, gc_stack, gc_cnt);
        write_sequnlock_bh(&base->lock);

        return p;
}
EXPORT_SYMBOL_GPL(inet_getpeer);

void inet_putpeer(struct inet_peer *p)
{
        /* The WRITE_ONCE() pairs with itself (we run lockless)
         * and the READ_ONCE() in inet_peer_gc()
         */
        WRITE_ONCE(p->dtime, (__u32)jiffies);

        if (refcount_dec_and_test(&p->refcnt))
                kfree_rcu(p, rcu);
}
EXPORT_SYMBOL_GPL(inet_putpeer);

/*
 *        Check transmit rate limitation for given message.
 *        The rate information is held in the inet_peer entries now.
 *        This function is generic and could be used for other purposes
 *        too. It uses a Token bucket filter as suggested by Alexey Kuznetsov.
 *
 *        Note that the same inet_peer fields are modified by functions in
 *        route.c too, but these work for packet destinations while xrlim_allow
 *        works for icmp destinations. This means the rate limiting information
 *        for one "ip object" is shared - and these ICMPs are twice limited:
 *        by source and by destination.
 *
 *        RFC 1812: 4.3.2.8 SHOULD be able to limit error message rate
 *                          SHOULD allow setting of rate limits
 *
 *         Shared between ICMPv4 and ICMPv6.
 */
#define XRLIM_BURST_FACTOR 6
bool inet_peer_xrlim_allow(struct inet_peer *peer, int timeout)
{
        unsigned long now, token;
        bool rc = false;

        if (!peer)
                return true;

        token = peer->rate_tokens;
        now = jiffies;
        token += now - peer->rate_last;
        peer->rate_last = now;
        if (token > XRLIM_BURST_FACTOR * timeout)
                token = XRLIM_BURST_FACTOR * timeout;
        if (token >= timeout) {
                token -= timeout;
                rc = true;
        }
        peer->rate_tokens = token;
        return rc;
}
EXPORT_SYMBOL(inet_peer_xrlim_allow);

void inetpeer_invalidate_tree(struct inet_peer_base *base)
{
        struct rb_node *p = rb_first(&base->rb_root);

        while (p) {
                struct inet_peer *peer = rb_entry(p, struct inet_peer, rb_node);

                p = rb_next(p);
                rb_erase(&peer->rb_node, &base->rb_root);
                inet_putpeer(peer);
                cond_resched();
        }

        base->total = 0;
}
EXPORT_SYMBOL(inetpeer_invalidate_tree);

























   23 































   36 
   36 
















































































   28 
















































    5 



    5 








   23 
































   31 



















   10 

   10 
    9 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_INODE_H
#define _BCACHEFS_INODE_H

#include "bkey.h"
#include "bkey_methods.h"
#include "opts.h"
#include "snapshot.h"

enum bch_validate_flags;
extern const char * const bch2_inode_opts[];

int bch2_inode_validate(struct bch_fs *, struct bkey_s_c,
                       enum bch_validate_flags);
int bch2_inode_v2_validate(struct bch_fs *, struct bkey_s_c,
                          enum bch_validate_flags);
int bch2_inode_v3_validate(struct bch_fs *, struct bkey_s_c,
                          enum bch_validate_flags);
void bch2_inode_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c);

int __bch2_inode_has_child_snapshots(struct btree_trans *, struct bpos);

static inline int bch2_inode_has_child_snapshots(struct btree_trans *trans, struct bpos pos)
{
        return bch2_snapshot_is_leaf(trans->c, pos.snapshot) <= 0
                ? __bch2_inode_has_child_snapshots(trans, pos)
                : 0;
}

int bch2_trigger_inode(struct btree_trans *, enum btree_id, unsigned,
                       struct bkey_s_c, struct bkey_s,
                       enum btree_iter_update_trigger_flags);

#define bch2_bkey_ops_inode ((struct bkey_ops) {        \
        .key_validate        = bch2_inode_validate,                \
        .val_to_text        = bch2_inode_to_text,                \
        .trigger        = bch2_trigger_inode,                \
        .min_val_size        = 16,                                \
})

#define bch2_bkey_ops_inode_v2 ((struct bkey_ops) {        \
        .key_validate        = bch2_inode_v2_validate,        \
        .val_to_text        = bch2_inode_to_text,                \
        .trigger        = bch2_trigger_inode,                \
        .min_val_size        = 32,                                \
})

#define bch2_bkey_ops_inode_v3 ((struct bkey_ops) {        \
        .key_validate        = bch2_inode_v3_validate,        \
        .val_to_text        = bch2_inode_to_text,                \
        .trigger        = bch2_trigger_inode,                \
        .min_val_size        = 48,                                \
})

static inline bool bkey_is_inode(const struct bkey *k)
{
        return  k->type == KEY_TYPE_inode ||
                k->type == KEY_TYPE_inode_v2 ||
                k->type == KEY_TYPE_inode_v3;
}

int bch2_inode_generation_validate(struct bch_fs *, struct bkey_s_c,
                                  enum bch_validate_flags);
void bch2_inode_generation_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c);

#define bch2_bkey_ops_inode_generation ((struct bkey_ops) {        \
        .key_validate        = bch2_inode_generation_validate,        \
        .val_to_text        = bch2_inode_generation_to_text,        \
        .min_val_size        = 8,                                        \
})

#if 0
typedef struct {
        u64                        lo;
        u32                        hi;
} __packed __aligned(4) u96;
#endif
typedef u64 u96;

struct bch_inode_unpacked {
        u64                        bi_inum;
        u32                        bi_snapshot;
        u64                        bi_journal_seq;
        __le64                        bi_hash_seed;
        u64                        bi_size;
        u64                        bi_sectors;
        u64                        bi_version;
        u32                        bi_flags;
        u16                        bi_mode;

#define x(_name, _bits)        u##_bits _name;
        BCH_INODE_FIELDS_v3()
#undef  x
};
BITMASK(INODE_STR_HASH,        struct bch_inode_unpacked, bi_flags, 20, 24);

struct bkey_inode_buf {
        struct bkey_i_inode_v3        inode;

#define x(_name, _bits)                + 8 + _bits / 8
        u8                _pad[0 + BCH_INODE_FIELDS_v3()];
#undef  x
};

void bch2_inode_pack(struct bkey_inode_buf *, const struct bch_inode_unpacked *);
int bch2_inode_unpack(struct bkey_s_c, struct bch_inode_unpacked *);
struct bkey_i *bch2_inode_to_v3(struct btree_trans *, struct bkey_i *);

void bch2_inode_unpacked_to_text(struct printbuf *, struct bch_inode_unpacked *);

int __bch2_inode_peek(struct btree_trans *, struct btree_iter *,
                      struct bch_inode_unpacked *, subvol_inum, unsigned, bool);

static inline int bch2_inode_peek_nowarn(struct btree_trans *trans,
                                         struct btree_iter *iter,
                                         struct bch_inode_unpacked *inode,
                                         subvol_inum inum, unsigned flags)
{
        return __bch2_inode_peek(trans, iter, inode, inum, flags, false);
}

static inline int bch2_inode_peek(struct btree_trans *trans,
                                  struct btree_iter *iter,
                                  struct bch_inode_unpacked *inode,
                                  subvol_inum inum, unsigned flags)
{
        return __bch2_inode_peek(trans, iter, inode, inum, flags, true);
        int ret = bch2_inode_peek_nowarn(trans, iter, inode, inum, flags);
        return ret;
}

int bch2_inode_write_flags(struct btree_trans *, struct btree_iter *,
                     struct bch_inode_unpacked *, enum btree_iter_update_trigger_flags);

static inline int bch2_inode_write(struct btree_trans *trans,
                     struct btree_iter *iter,
                     struct bch_inode_unpacked *inode)
{
        return bch2_inode_write_flags(trans, iter, inode, 0);
}

int __bch2_fsck_write_inode(struct btree_trans *, struct bch_inode_unpacked *);
int bch2_fsck_write_inode(struct btree_trans *, struct bch_inode_unpacked *);

void bch2_inode_init_early(struct bch_fs *,
                           struct bch_inode_unpacked *);
void bch2_inode_init_late(struct bch_inode_unpacked *, u64,
                          uid_t, gid_t, umode_t, dev_t,
                          struct bch_inode_unpacked *);
void bch2_inode_init(struct bch_fs *, struct bch_inode_unpacked *,
                     uid_t, gid_t, umode_t, dev_t,
                     struct bch_inode_unpacked *);

int bch2_inode_create(struct btree_trans *, struct btree_iter *,
                      struct bch_inode_unpacked *, u32, u64);

int bch2_inode_rm(struct bch_fs *, subvol_inum);

int bch2_inode_find_by_inum_nowarn_trans(struct btree_trans *,
                                  subvol_inum,
                                  struct bch_inode_unpacked *);
int bch2_inode_find_by_inum_trans(struct btree_trans *, subvol_inum,
                                  struct bch_inode_unpacked *);
int bch2_inode_find_by_inum(struct bch_fs *, subvol_inum,
                            struct bch_inode_unpacked *);

#define inode_opt_get(_c, _inode, _name)                        \
        ((_inode)->bi_##_name ? (_inode)->bi_##_name - 1 : (_c)->opts._name)

static inline void bch2_inode_opt_set(struct bch_inode_unpacked *inode,
                                      enum inode_opt_id id, u64 v)
{
        switch (id) {
#define x(_name, ...)                                                        \
        case Inode_opt_##_name:                                                \
                inode->bi_##_name = v;                                        \
                break;
        BCH_INODE_OPTS()
#undef x
        default:
                BUG();
        }
}

static inline u64 bch2_inode_opt_get(struct bch_inode_unpacked *inode,
                                     enum inode_opt_id id)
{
        switch (id) {
#define x(_name, ...)                                                        \
        case Inode_opt_##_name:                                                \
                return inode->bi_##_name;
        BCH_INODE_OPTS()
#undef x
        default:
                BUG();
        }
}

static inline u8 mode_to_type(umode_t mode)
{
        return (mode >> 12) & 15;
}

static inline u8 inode_d_type(struct bch_inode_unpacked *inode)
{
        return inode->bi_subvol ? DT_SUBVOL : mode_to_type(inode->bi_mode);
}

static inline u32 bch2_inode_flags(struct bkey_s_c k)
{
        switch (k.k->type) {
        case KEY_TYPE_inode:
                return le32_to_cpu(bkey_s_c_to_inode(k).v->bi_flags);
        case KEY_TYPE_inode_v2:
                return le64_to_cpu(bkey_s_c_to_inode_v2(k).v->bi_flags);
        case KEY_TYPE_inode_v3:
                return le64_to_cpu(bkey_s_c_to_inode_v3(k).v->bi_flags);
        default:
                return 0;
        }
}

/* i_nlink: */

static inline unsigned nlink_bias(umode_t mode)
{
        return S_ISDIR(mode) ? 2 : 1;
}

static inline unsigned bch2_inode_nlink_get(struct bch_inode_unpacked *bi)
{
        return bi->bi_flags & BCH_INODE_unlinked
                  ? 0
                  : bi->bi_nlink + nlink_bias(bi->bi_mode);
}

static inline void bch2_inode_nlink_set(struct bch_inode_unpacked *bi,
                                        unsigned nlink)
{
        if (nlink) {
                bi->bi_nlink = nlink - nlink_bias(bi->bi_mode);
                bi->bi_flags &= ~BCH_INODE_unlinked;
        } else {
                bi->bi_nlink = 0;
                bi->bi_flags |= BCH_INODE_unlinked;
        }
}

int bch2_inode_nlink_inc(struct bch_inode_unpacked *);
void bch2_inode_nlink_dec(struct btree_trans *, struct bch_inode_unpacked *);

static inline bool bch2_inode_should_have_bp(struct bch_inode_unpacked *inode)
{
        bool inode_has_bp = inode->bi_dir || inode->bi_dir_offset;

        return S_ISDIR(inode->bi_mode) ||
                (!inode->bi_nlink && inode_has_bp);
}

struct bch_opts bch2_inode_opts_to_opts(struct bch_inode_unpacked *);
void bch2_inode_opts_get(struct bch_io_opts *, struct bch_fs *,
                         struct bch_inode_unpacked *);
int bch2_inum_opts_get(struct btree_trans*, subvol_inum, struct bch_io_opts *);

int bch2_inode_rm_snapshot(struct btree_trans *, u64, u32);
int bch2_delete_dead_inodes(struct bch_fs *);

#endif /* _BCACHEFS_INODE_H */















































































































    1 


    1 
    1 








    6 




    5 






    2 



    1 

    2 


    1 

    6 
    4 
    3 
    1 

    1 


    1 












    1 


































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * PTP 1588 clock support
 *
 * Copyright (C) 2010 OMICRON electronics GmbH
 */
#include <linux/device.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/posix-clock.h>
#include <linux/pps_kernel.h>
#include <linux/slab.h>
#include <linux/syscalls.h>
#include <linux/uaccess.h>
#include <linux/debugfs.h>
#include <linux/xarray.h>
#include <uapi/linux/sched/types.h>

#include "ptp_private.h"

#define PTP_MAX_ALARMS 4
#define PTP_PPS_DEFAULTS (PPS_CAPTUREASSERT | PPS_OFFSETASSERT)
#define PTP_PPS_EVENT PPS_CAPTUREASSERT
#define PTP_PPS_MODE (PTP_PPS_DEFAULTS | PPS_CANWAIT | PPS_TSFMT_TSPEC)

const struct class ptp_class = {
        .name = "ptp",
        .dev_groups = ptp_groups
};

/* private globals */

static dev_t ptp_devt;

static DEFINE_XARRAY_ALLOC(ptp_clocks_map);

/* time stamp event queue operations */

static inline int queue_free(struct timestamp_event_queue *q)
{
        return PTP_MAX_TIMESTAMPS - queue_cnt(q) - 1;
}

static void enqueue_external_timestamp(struct timestamp_event_queue *queue,
                                       struct ptp_clock_event *src)
{
        struct ptp_extts_event *dst;
        struct timespec64 offset_ts;
        unsigned long flags;
        s64 seconds;
        u32 remainder;

        if (src->type == PTP_CLOCK_EXTTS) {
                seconds = div_u64_rem(src->timestamp, 1000000000, &remainder);
        } else if (src->type == PTP_CLOCK_EXTOFF) {
                offset_ts = ns_to_timespec64(src->offset);
                seconds = offset_ts.tv_sec;
                remainder = offset_ts.tv_nsec;
        } else {
                WARN(1, "%s: unknown type %d\n", __func__, src->type);
                return;
        }

        spin_lock_irqsave(&queue->lock, flags);

        dst = &queue->buf[queue->tail];
        dst->index = src->index;
        dst->flags = PTP_EXTTS_EVENT_VALID;
        dst->t.sec = seconds;
        dst->t.nsec = remainder;
        if (src->type == PTP_CLOCK_EXTOFF)
                dst->flags |= PTP_EXT_OFFSET;

        /* Both WRITE_ONCE() are paired with READ_ONCE() in queue_cnt() */
        if (!queue_free(queue))
                WRITE_ONCE(queue->head, (queue->head + 1) % PTP_MAX_TIMESTAMPS);

        WRITE_ONCE(queue->tail, (queue->tail + 1) % PTP_MAX_TIMESTAMPS);

        spin_unlock_irqrestore(&queue->lock, flags);
}

/* posix clock implementation */

static int ptp_clock_getres(struct posix_clock *pc, struct timespec64 *tp)
{
        tp->tv_sec = 0;
        tp->tv_nsec = 1;
        return 0;
}

static int ptp_clock_settime(struct posix_clock *pc, const struct timespec64 *tp)
{
        struct ptp_clock *ptp = container_of(pc, struct ptp_clock, clock);

        if (ptp_clock_freerun(ptp)) {
                pr_err("ptp: physical clock is free running\n");
                return -EBUSY;
        }

        return  ptp->info->settime64(ptp->info, tp);
}

static int ptp_clock_gettime(struct posix_clock *pc, struct timespec64 *tp)
{
        struct ptp_clock *ptp = container_of(pc, struct ptp_clock, clock);
        int err;

        if (ptp->info->gettimex64)
                err = ptp->info->gettimex64(ptp->info, tp, NULL);
        else
                err = ptp->info->gettime64(ptp->info, tp);
        return err;
}

static int ptp_clock_adjtime(struct posix_clock *pc, struct __kernel_timex *tx)
{
        struct ptp_clock *ptp = container_of(pc, struct ptp_clock, clock);
        struct ptp_clock_info *ops;
        int err = -EOPNOTSUPP;

        if (ptp_clock_freerun(ptp)) {
                pr_err("ptp: physical clock is free running\n");
                return -EBUSY;
        }

        ops = ptp->info;

        if (tx->modes & ADJ_SETOFFSET) {
                struct timespec64 ts;
                ktime_t kt;
                s64 delta;

                ts.tv_sec  = tx->time.tv_sec;
                ts.tv_nsec = tx->time.tv_usec;

                if (!(tx->modes & ADJ_NANO))
                        ts.tv_nsec *= 1000;

                if ((unsigned long) ts.tv_nsec >= NSEC_PER_SEC)
                        return -EINVAL;

                kt = timespec64_to_ktime(ts);
                delta = ktime_to_ns(kt);
                err = ops->adjtime(ops, delta);
        } else if (tx->modes & ADJ_FREQUENCY) {
                long ppb = scaled_ppm_to_ppb(tx->freq);
                if (ppb > ops->max_adj || ppb < -ops->max_adj)
                        return -ERANGE;
                err = ops->adjfine(ops, tx->freq);
                if (!err)
                        ptp->dialed_frequency = tx->freq;
        } else if (tx->modes & ADJ_OFFSET) {
                if (ops->adjphase) {
                        s32 max_phase_adj = ops->getmaxphase(ops);
                        s32 offset = tx->offset;

                        if (!(tx->modes & ADJ_NANO))
                                offset *= NSEC_PER_USEC;

                        if (offset > max_phase_adj || offset < -max_phase_adj)
                                return -ERANGE;

                        err = ops->adjphase(ops, offset);
                }
        } else if (tx->modes == 0) {
                tx->freq = ptp->dialed_frequency;
                err = 0;
        }

        return err;
}

static struct posix_clock_operations ptp_clock_ops = {
        .owner                = THIS_MODULE,
        .clock_adjtime        = ptp_clock_adjtime,
        .clock_gettime        = ptp_clock_gettime,
        .clock_getres        = ptp_clock_getres,
        .clock_settime        = ptp_clock_settime,
        .ioctl                = ptp_ioctl,
        .open                = ptp_open,
        .release        = ptp_release,
        .poll                = ptp_poll,
        .read                = ptp_read,
};

static void ptp_clock_release(struct device *dev)
{
        struct ptp_clock *ptp = container_of(dev, struct ptp_clock, dev);
        struct timestamp_event_queue *tsevq;
        unsigned long flags;

        ptp_cleanup_pin_groups(ptp);
        kfree(ptp->vclock_index);
        mutex_destroy(&ptp->pincfg_mux);
        mutex_destroy(&ptp->n_vclocks_mux);
        /* Delete first entry */
        spin_lock_irqsave(&ptp->tsevqs_lock, flags);
        tsevq = list_first_entry(&ptp->tsevqs, struct timestamp_event_queue,
                                 qlist);
        list_del(&tsevq->qlist);
        spin_unlock_irqrestore(&ptp->tsevqs_lock, flags);
        bitmap_free(tsevq->mask);
        kfree(tsevq);
        debugfs_remove(ptp->debugfs_root);
        xa_erase(&ptp_clocks_map, ptp->index);
        kfree(ptp);
}

static int ptp_getcycles64(struct ptp_clock_info *info, struct timespec64 *ts)
{
        if (info->getcyclesx64)
                return info->getcyclesx64(info, ts, NULL);
        else
                return info->gettime64(info, ts);
}

static void ptp_aux_kworker(struct kthread_work *work)
{
        struct ptp_clock *ptp = container_of(work, struct ptp_clock,
                                             aux_work.work);
        struct ptp_clock_info *info = ptp->info;
        long delay;

        delay = info->do_aux_work(info);

        if (delay >= 0)
                kthread_queue_delayed_work(ptp->kworker, &ptp->aux_work, delay);
}

/* public interface */

struct ptp_clock *ptp_clock_register(struct ptp_clock_info *info,
                                     struct device *parent)
{
        struct ptp_clock *ptp;
        struct timestamp_event_queue *queue = NULL;
        int err, index, major = MAJOR(ptp_devt);
        char debugfsname[16];
        size_t size;

        if (info->n_alarm > PTP_MAX_ALARMS)
                return ERR_PTR(-EINVAL);

        /* Initialize a clock structure. */
        ptp = kzalloc(sizeof(struct ptp_clock), GFP_KERNEL);
        if (!ptp) {
                err = -ENOMEM;
                goto no_memory;
        }

        err = xa_alloc(&ptp_clocks_map, &index, ptp, xa_limit_31b,
                       GFP_KERNEL);
        if (err)
                goto no_slot;

        ptp->clock.ops = ptp_clock_ops;
        ptp->info = info;
        ptp->devid = MKDEV(major, index);
        ptp->index = index;
        INIT_LIST_HEAD(&ptp->tsevqs);
        queue = kzalloc(sizeof(*queue), GFP_KERNEL);
        if (!queue) {
                err = -ENOMEM;
                goto no_memory_queue;
        }
        list_add_tail(&queue->qlist, &ptp->tsevqs);
        spin_lock_init(&ptp->tsevqs_lock);
        queue->mask = bitmap_alloc(PTP_MAX_CHANNELS, GFP_KERNEL);
        if (!queue->mask) {
                err = -ENOMEM;
                goto no_memory_bitmap;
        }
        bitmap_set(queue->mask, 0, PTP_MAX_CHANNELS);
        spin_lock_init(&queue->lock);
        mutex_init(&ptp->pincfg_mux);
        mutex_init(&ptp->n_vclocks_mux);
        init_waitqueue_head(&ptp->tsev_wq);

        if (ptp->info->getcycles64 || ptp->info->getcyclesx64) {
                ptp->has_cycles = true;
                if (!ptp->info->getcycles64 && ptp->info->getcyclesx64)
                        ptp->info->getcycles64 = ptp_getcycles64;
        } else {
                /* Free running cycle counter not supported, use time. */
                ptp->info->getcycles64 = ptp_getcycles64;

                if (ptp->info->gettimex64)
                        ptp->info->getcyclesx64 = ptp->info->gettimex64;

                if (ptp->info->getcrosststamp)
                        ptp->info->getcrosscycles = ptp->info->getcrosststamp;
        }

        if (ptp->info->do_aux_work) {
                kthread_init_delayed_work(&ptp->aux_work, ptp_aux_kworker);
                ptp->kworker = kthread_create_worker(0, "ptp%d", ptp->index);
                if (IS_ERR(ptp->kworker)) {
                        err = PTR_ERR(ptp->kworker);
                        pr_err("failed to create ptp aux_worker %d\n", err);
                        goto kworker_err;
                }
        }

        /* PTP virtual clock is being registered under physical clock */
        if (parent && parent->class && parent->class->name &&
            strcmp(parent->class->name, "ptp") == 0)
                ptp->is_virtual_clock = true;

        if (!ptp->is_virtual_clock) {
                ptp->max_vclocks = PTP_DEFAULT_MAX_VCLOCKS;

                size = sizeof(int) * ptp->max_vclocks;
                ptp->vclock_index = kzalloc(size, GFP_KERNEL);
                if (!ptp->vclock_index) {
                        err = -ENOMEM;
                        goto no_mem_for_vclocks;
                }
        }

        err = ptp_populate_pin_groups(ptp);
        if (err)
                goto no_pin_groups;

        /* Register a new PPS source. */
        if (info->pps) {
                struct pps_source_info pps;
                memset(&pps, 0, sizeof(pps));
                snprintf(pps.name, PPS_MAX_NAME_LEN, "ptp%d", index);
                pps.mode = PTP_PPS_MODE;
                pps.owner = info->owner;
                ptp->pps_source = pps_register_source(&pps, PTP_PPS_DEFAULTS);
                if (IS_ERR(ptp->pps_source)) {
                        err = PTR_ERR(ptp->pps_source);
                        pr_err("failed to register pps source\n");
                        goto no_pps;
                }
                ptp->pps_source->lookup_cookie = ptp;
        }

        /* Initialize a new device of our class in our clock structure. */
        device_initialize(&ptp->dev);
        ptp->dev.devt = ptp->devid;
        ptp->dev.class = &ptp_class;
        ptp->dev.parent = parent;
        ptp->dev.groups = ptp->pin_attr_groups;
        ptp->dev.release = ptp_clock_release;
        dev_set_drvdata(&ptp->dev, ptp);
        dev_set_name(&ptp->dev, "ptp%d", ptp->index);

        /* Create a posix clock and link it to the device. */
        err = posix_clock_register(&ptp->clock, &ptp->dev);
        if (err) {
                if (ptp->pps_source)
                        pps_unregister_source(ptp->pps_source);

                if (ptp->kworker)
                        kthread_destroy_worker(ptp->kworker);

                put_device(&ptp->dev);

                pr_err("failed to create posix clock\n");
                return ERR_PTR(err);
        }

        /* Debugfs initialization */
        snprintf(debugfsname, sizeof(debugfsname), "ptp%d", ptp->index);
        ptp->debugfs_root = debugfs_create_dir(debugfsname, NULL);

        return ptp;

no_pps:
        ptp_cleanup_pin_groups(ptp);
no_pin_groups:
        kfree(ptp->vclock_index);
no_mem_for_vclocks:
        if (ptp->kworker)
                kthread_destroy_worker(ptp->kworker);
kworker_err:
        mutex_destroy(&ptp->pincfg_mux);
        mutex_destroy(&ptp->n_vclocks_mux);
        bitmap_free(queue->mask);
no_memory_bitmap:
        list_del(&queue->qlist);
        kfree(queue);
no_memory_queue:
        xa_erase(&ptp_clocks_map, index);
no_slot:
        kfree(ptp);
no_memory:
        return ERR_PTR(err);
}
EXPORT_SYMBOL(ptp_clock_register);

static int unregister_vclock(struct device *dev, void *data)
{
        struct ptp_clock *ptp = dev_get_drvdata(dev);

        ptp_vclock_unregister(info_to_vclock(ptp->info));
        return 0;
}

int ptp_clock_unregister(struct ptp_clock *ptp)
{
        if (ptp_vclock_in_use(ptp)) {
                device_for_each_child(&ptp->dev, NULL, unregister_vclock);
        }

        ptp->defunct = 1;
        wake_up_interruptible(&ptp->tsev_wq);

        if (ptp->kworker) {
                kthread_cancel_delayed_work_sync(&ptp->aux_work);
                kthread_destroy_worker(ptp->kworker);
        }

        /* Release the clock's resources. */
        if (ptp->pps_source)
                pps_unregister_source(ptp->pps_source);

        posix_clock_unregister(&ptp->clock);

        return 0;
}
EXPORT_SYMBOL(ptp_clock_unregister);

void ptp_clock_event(struct ptp_clock *ptp, struct ptp_clock_event *event)
{
        struct timestamp_event_queue *tsevq;
        struct pps_event_time evt;
        unsigned long flags;

        switch (event->type) {

        case PTP_CLOCK_ALARM:
                break;

        case PTP_CLOCK_EXTTS:
        case PTP_CLOCK_EXTOFF:
                /* Enqueue timestamp on selected queues */
                spin_lock_irqsave(&ptp->tsevqs_lock, flags);
                list_for_each_entry(tsevq, &ptp->tsevqs, qlist) {
                        if (test_bit((unsigned int)event->index, tsevq->mask))
                                enqueue_external_timestamp(tsevq, event);
                }
                spin_unlock_irqrestore(&ptp->tsevqs_lock, flags);
                wake_up_interruptible(&ptp->tsev_wq);
                break;

        case PTP_CLOCK_PPS:
                pps_get_ts(&evt);
                pps_event(ptp->pps_source, &evt, PTP_PPS_EVENT, NULL);
                break;

        case PTP_CLOCK_PPSUSR:
                pps_event(ptp->pps_source, &event->pps_times,
                          PTP_PPS_EVENT, NULL);
                break;
        }
}
EXPORT_SYMBOL(ptp_clock_event);

int ptp_clock_index(struct ptp_clock *ptp)
{
        return ptp->index;
}
EXPORT_SYMBOL(ptp_clock_index);

int ptp_find_pin(struct ptp_clock *ptp,
                 enum ptp_pin_function func, unsigned int chan)
{
        struct ptp_pin_desc *pin = NULL;
        int i;

        for (i = 0; i < ptp->info->n_pins; i++) {
                if (ptp->info->pin_config[i].func == func &&
                    ptp->info->pin_config[i].chan == chan) {
                        pin = &ptp->info->pin_config[i];
                        break;
                }
        }

        return pin ? i : -1;
}
EXPORT_SYMBOL(ptp_find_pin);

int ptp_find_pin_unlocked(struct ptp_clock *ptp,
                          enum ptp_pin_function func, unsigned int chan)
{
        int result;

        mutex_lock(&ptp->pincfg_mux);

        result = ptp_find_pin(ptp, func, chan);

        mutex_unlock(&ptp->pincfg_mux);

        return result;
}
EXPORT_SYMBOL(ptp_find_pin_unlocked);

int ptp_schedule_worker(struct ptp_clock *ptp, unsigned long delay)
{
        return kthread_mod_delayed_work(ptp->kworker, &ptp->aux_work, delay);
}
EXPORT_SYMBOL(ptp_schedule_worker);

void ptp_cancel_worker_sync(struct ptp_clock *ptp)
{
        kthread_cancel_delayed_work_sync(&ptp->aux_work);
}
EXPORT_SYMBOL(ptp_cancel_worker_sync);

/* module operations */

static void __exit ptp_exit(void)
{
        class_unregister(&ptp_class);
        unregister_chrdev_region(ptp_devt, MINORMASK + 1);
        xa_destroy(&ptp_clocks_map);
}

static int __init ptp_init(void)
{
        int err;

        err = class_register(&ptp_class);
        if (err) {
                pr_err("ptp: failed to allocate class\n");
                return err;
        }

        err = alloc_chrdev_region(&ptp_devt, 0, MINORMASK + 1, "ptp");
        if (err < 0) {
                pr_err("ptp: failed to allocate device region\n");
                goto no_region;
        }

        pr_info("PTP clock support registered\n");
        return 0;

no_region:
        class_unregister(&ptp_class);
        return err;
}

subsys_initcall(ptp_init);
module_exit(ptp_exit);

MODULE_AUTHOR("Richard Cochran <richardcochran@gmail.com>");
MODULE_DESCRIPTION("PTP clocks support");
MODULE_LICENSE("GPL");


































































   25 










   24 








   24 




   23 


   24 
   24 






   18 








   17 
   16 
   15 


   14 



    2 
    5 


    3 






    5 
    3 



    3 



   18 








   14 







   13 






   13 




   12 




   12 



    1 









    2 





    2 




    2 

    2 
    2 
    2 



    2 
    2 


    2 



    2 







    2 






    2 









    2 
    2 
    1 
    1 







    1 





    2 
    2 
    2 
    2 






    4 





    2 





    2 

    1 




    1 


    1 







    1 






    1 
    4 













    2 

    2 

    1 



    1 






    3 





    1 

    1 



    2 


    1 


    1 
    3 















   14 
   13 


   12 








   11 

   14 



















    8 





    8 


    8 
    8 


    7 



    8 





















   11 


    9 






    8 

    1 


    1 



    1 



    1 



    1 



    1 




    1 



    1 






    8 


    8 

   11 

    8 










    8 





    3 












    3 

























































   20 































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * (C) 2012 by Pablo Neira Ayuso <pablo@netfilter.org>
 * (C) 2012 by Vyatta Inc. <http://www.vyatta.com>
 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/rculist.h>
#include <linux/rculist_nulls.h>
#include <linux/types.h>
#include <linux/timer.h>
#include <linux/security.h>
#include <linux/skbuff.h>
#include <linux/errno.h>
#include <linux/netlink.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/slab.h>

#include <linux/netfilter.h>
#include <net/netlink.h>
#include <net/netns/generic.h>
#include <net/sock.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_core.h>
#include <net/netfilter/nf_conntrack_l4proto.h>
#include <net/netfilter/nf_conntrack_tuple.h>
#include <net/netfilter/nf_conntrack_timeout.h>

#include <linux/netfilter/nfnetlink.h>
#include <linux/netfilter/nfnetlink_cttimeout.h>

static unsigned int nfct_timeout_id __read_mostly;

struct ctnl_timeout {
        struct list_head        head;
        struct list_head        free_head;
        struct rcu_head                rcu_head;
        refcount_t                refcnt;
        char                        name[CTNL_TIMEOUT_NAME_MAX];

        /* must be at the end */
        struct nf_ct_timeout        timeout;
};

struct nfct_timeout_pernet {
        struct list_head        nfct_timeout_list;
        struct list_head        nfct_timeout_freelist;
};

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Pablo Neira Ayuso <pablo@netfilter.org>");
MODULE_DESCRIPTION("cttimeout: Extended Netfilter Connection Tracking timeout tuning");

static const struct nla_policy cttimeout_nla_policy[CTA_TIMEOUT_MAX+1] = {
        [CTA_TIMEOUT_NAME]        = { .type = NLA_NUL_STRING,
                                    .len  = CTNL_TIMEOUT_NAME_MAX - 1},
        [CTA_TIMEOUT_L3PROTO]        = { .type = NLA_U16 },
        [CTA_TIMEOUT_L4PROTO]        = { .type = NLA_U8 },
        [CTA_TIMEOUT_DATA]        = { .type = NLA_NESTED },
};

static struct nfct_timeout_pernet *nfct_timeout_pernet(struct net *net)
{
        return net_generic(net, nfct_timeout_id);
}

static int
ctnl_timeout_parse_policy(void *timeout,
                          const struct nf_conntrack_l4proto *l4proto,
                          struct net *net, const struct nlattr *attr)
{
        struct nlattr **tb;
        int ret = 0;

        tb = kcalloc(l4proto->ctnl_timeout.nlattr_max + 1, sizeof(*tb),
                     GFP_KERNEL);

        if (!tb)
                return -ENOMEM;

        ret = nla_parse_nested_deprecated(tb,
                                          l4proto->ctnl_timeout.nlattr_max,
                                          attr,
                                          l4proto->ctnl_timeout.nla_policy,
                                          NULL);
        if (ret < 0)
                goto err;

        ret = l4proto->ctnl_timeout.nlattr_to_obj(tb, net, timeout);

err:
        kfree(tb);
        return ret;
}

static int cttimeout_new_timeout(struct sk_buff *skb,
                                 const struct nfnl_info *info,
                                 const struct nlattr * const cda[])
{
        struct nfct_timeout_pernet *pernet = nfct_timeout_pernet(info->net);
        __u16 l3num;
        __u8 l4num;
        const struct nf_conntrack_l4proto *l4proto;
        struct ctnl_timeout *timeout, *matching = NULL;
        char *name;
        int ret;

        if (!cda[CTA_TIMEOUT_NAME] ||
            !cda[CTA_TIMEOUT_L3PROTO] ||
            !cda[CTA_TIMEOUT_L4PROTO] ||
            !cda[CTA_TIMEOUT_DATA])
                return -EINVAL;

        name = nla_data(cda[CTA_TIMEOUT_NAME]);
        l3num = ntohs(nla_get_be16(cda[CTA_TIMEOUT_L3PROTO]));
        l4num = nla_get_u8(cda[CTA_TIMEOUT_L4PROTO]);

        list_for_each_entry(timeout, &pernet->nfct_timeout_list, head) {
                if (strncmp(timeout->name, name, CTNL_TIMEOUT_NAME_MAX) != 0)
                        continue;

                if (info->nlh->nlmsg_flags & NLM_F_EXCL)
                        return -EEXIST;

                matching = timeout;
                break;
        }

        if (matching) {
                if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
                        /* You cannot replace one timeout policy by another of
                         * different kind, sorry.
                         */
                        if (matching->timeout.l3num != l3num ||
                            matching->timeout.l4proto->l4proto != l4num)
                                return -EINVAL;

                        return ctnl_timeout_parse_policy(&matching->timeout.data,
                                                         matching->timeout.l4proto,
                                                         info->net,
                                                         cda[CTA_TIMEOUT_DATA]);
                }

                return -EBUSY;
        }

        l4proto = nf_ct_l4proto_find(l4num);

        /* This protocol is not supportted, skip. */
        if (l4proto->l4proto != l4num) {
                ret = -EOPNOTSUPP;
                goto err_proto_put;
        }

        timeout = kzalloc(sizeof(struct ctnl_timeout) +
                          l4proto->ctnl_timeout.obj_size, GFP_KERNEL);
        if (timeout == NULL) {
                ret = -ENOMEM;
                goto err_proto_put;
        }

        ret = ctnl_timeout_parse_policy(&timeout->timeout.data, l4proto,
                                        info->net, cda[CTA_TIMEOUT_DATA]);
        if (ret < 0)
                goto err;

        strcpy(timeout->name, nla_data(cda[CTA_TIMEOUT_NAME]));
        timeout->timeout.l3num = l3num;
        timeout->timeout.l4proto = l4proto;
        refcount_set(&timeout->refcnt, 1);
        __module_get(THIS_MODULE);
        list_add_tail_rcu(&timeout->head, &pernet->nfct_timeout_list);

        return 0;
err:
        kfree(timeout);
err_proto_put:
        return ret;
}

static int
ctnl_timeout_fill_info(struct sk_buff *skb, u32 portid, u32 seq, u32 type,
                       int event, struct ctnl_timeout *timeout)
{
        struct nlmsghdr *nlh;
        unsigned int flags = portid ? NLM_F_MULTI : 0;
        const struct nf_conntrack_l4proto *l4proto = timeout->timeout.l4proto;
        struct nlattr *nest_parms;
        int ret;

        event = nfnl_msg_type(NFNL_SUBSYS_CTNETLINK_TIMEOUT, event);
        nlh = nfnl_msg_put(skb, portid, seq, event, flags, AF_UNSPEC,
                           NFNETLINK_V0, 0);
        if (!nlh)
                goto nlmsg_failure;

        if (nla_put_string(skb, CTA_TIMEOUT_NAME, timeout->name) ||
            nla_put_be16(skb, CTA_TIMEOUT_L3PROTO,
                         htons(timeout->timeout.l3num)) ||
            nla_put_u8(skb, CTA_TIMEOUT_L4PROTO, l4proto->l4proto) ||
            nla_put_be32(skb, CTA_TIMEOUT_USE,
                         htonl(refcount_read(&timeout->refcnt))))
                goto nla_put_failure;

        nest_parms = nla_nest_start(skb, CTA_TIMEOUT_DATA);
        if (!nest_parms)
                goto nla_put_failure;

        ret = l4proto->ctnl_timeout.obj_to_nlattr(skb, &timeout->timeout.data);
        if (ret < 0)
                goto nla_put_failure;

        nla_nest_end(skb, nest_parms);

        nlmsg_end(skb, nlh);
        return skb->len;

nlmsg_failure:
nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -1;
}

static int
ctnl_timeout_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct nfct_timeout_pernet *pernet;
        struct net *net = sock_net(skb->sk);
        struct ctnl_timeout *cur, *last;

        if (cb->args[2])
                return 0;

        last = (struct ctnl_timeout *)cb->args[1];
        if (cb->args[1])
                cb->args[1] = 0;

        rcu_read_lock();
        pernet = nfct_timeout_pernet(net);
        list_for_each_entry_rcu(cur, &pernet->nfct_timeout_list, head) {
                if (last) {
                        if (cur != last)
                                continue;

                        last = NULL;
                }
                if (ctnl_timeout_fill_info(skb, NETLINK_CB(cb->skb).portid,
                                           cb->nlh->nlmsg_seq,
                                           NFNL_MSG_TYPE(cb->nlh->nlmsg_type),
                                           IPCTNL_MSG_TIMEOUT_NEW, cur) < 0) {
                        cb->args[1] = (unsigned long)cur;
                        break;
                }
        }
        if (!cb->args[1])
                cb->args[2] = 1;
        rcu_read_unlock();
        return skb->len;
}

static int cttimeout_get_timeout(struct sk_buff *skb,
                                 const struct nfnl_info *info,
                                 const struct nlattr * const cda[])
{
        struct nfct_timeout_pernet *pernet = nfct_timeout_pernet(info->net);
        int ret = -ENOENT;
        char *name;
        struct ctnl_timeout *cur;

        if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
                struct netlink_dump_control c = {
                        .dump = ctnl_timeout_dump,
                };
                return netlink_dump_start(info->sk, skb, info->nlh, &c);
        }

        if (!cda[CTA_TIMEOUT_NAME])
                return -EINVAL;
        name = nla_data(cda[CTA_TIMEOUT_NAME]);

        list_for_each_entry(cur, &pernet->nfct_timeout_list, head) {
                struct sk_buff *skb2;

                if (strncmp(cur->name, name, CTNL_TIMEOUT_NAME_MAX) != 0)
                        continue;

                skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
                if (skb2 == NULL) {
                        ret = -ENOMEM;
                        break;
                }

                ret = ctnl_timeout_fill_info(skb2, NETLINK_CB(skb).portid,
                                             info->nlh->nlmsg_seq,
                                             NFNL_MSG_TYPE(info->nlh->nlmsg_type),
                                             IPCTNL_MSG_TIMEOUT_NEW, cur);
                if (ret <= 0) {
                        kfree_skb(skb2);
                        break;
                }

                ret = nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
                break;
        }

        return ret;
}

/* try to delete object, fail if it is still in use. */
static int ctnl_timeout_try_del(struct net *net, struct ctnl_timeout *timeout)
{
        int ret = 0;

        /* We want to avoid races with ctnl_timeout_put. So only when the
         * current refcnt is 1, we decrease it to 0.
         */
        if (refcount_dec_if_one(&timeout->refcnt)) {
                /* We are protected by nfnl mutex. */
                list_del_rcu(&timeout->head);
                nf_ct_untimeout(net, &timeout->timeout);
                kfree_rcu(timeout, rcu_head);
        } else {
                ret = -EBUSY;
        }
        return ret;
}

static int cttimeout_del_timeout(struct sk_buff *skb,
                                 const struct nfnl_info *info,
                                 const struct nlattr * const cda[])
{
        struct nfct_timeout_pernet *pernet = nfct_timeout_pernet(info->net);
        struct ctnl_timeout *cur, *tmp;
        int ret = -ENOENT;
        char *name;

        if (!cda[CTA_TIMEOUT_NAME]) {
                list_for_each_entry_safe(cur, tmp, &pernet->nfct_timeout_list,
                                         head)
                        ctnl_timeout_try_del(info->net, cur);

                return 0;
        }
        name = nla_data(cda[CTA_TIMEOUT_NAME]);

        list_for_each_entry(cur, &pernet->nfct_timeout_list, head) {
                if (strncmp(cur->name, name, CTNL_TIMEOUT_NAME_MAX) != 0)
                        continue;

                ret = ctnl_timeout_try_del(info->net, cur);
                if (ret < 0)
                        return ret;

                break;
        }
        return ret;
}

static int cttimeout_default_set(struct sk_buff *skb,
                                 const struct nfnl_info *info,
                                 const struct nlattr * const cda[])
{
        const struct nf_conntrack_l4proto *l4proto;
        __u8 l4num;
        int ret;

        if (!cda[CTA_TIMEOUT_L4PROTO] ||
            !cda[CTA_TIMEOUT_DATA])
                return -EINVAL;

        l4num = nla_get_u8(cda[CTA_TIMEOUT_L4PROTO]);
        l4proto = nf_ct_l4proto_find(l4num);

        /* This protocol is not supported, skip. */
        if (l4proto->l4proto != l4num) {
                ret = -EOPNOTSUPP;
                goto err;
        }

        ret = ctnl_timeout_parse_policy(NULL, l4proto, info->net,
                                        cda[CTA_TIMEOUT_DATA]);
        if (ret < 0)
                goto err;

        return 0;
err:
        return ret;
}

static int
cttimeout_default_fill_info(struct net *net, struct sk_buff *skb, u32 portid,
                            u32 seq, u32 type, int event, u16 l3num,
                            const struct nf_conntrack_l4proto *l4proto,
                            const unsigned int *timeouts)
{
        struct nlmsghdr *nlh;
        unsigned int flags = portid ? NLM_F_MULTI : 0;
        struct nlattr *nest_parms;
        int ret;

        event = nfnl_msg_type(NFNL_SUBSYS_CTNETLINK_TIMEOUT, event);
        nlh = nfnl_msg_put(skb, portid, seq, event, flags, AF_UNSPEC,
                           NFNETLINK_V0, 0);
        if (!nlh)
                goto nlmsg_failure;

        if (nla_put_be16(skb, CTA_TIMEOUT_L3PROTO, htons(l3num)) ||
            nla_put_u8(skb, CTA_TIMEOUT_L4PROTO, l4proto->l4proto))
                goto nla_put_failure;

        nest_parms = nla_nest_start(skb, CTA_TIMEOUT_DATA);
        if (!nest_parms)
                goto nla_put_failure;

        ret = l4proto->ctnl_timeout.obj_to_nlattr(skb, timeouts);
        if (ret < 0)
                goto nla_put_failure;

        nla_nest_end(skb, nest_parms);

        nlmsg_end(skb, nlh);
        return skb->len;

nlmsg_failure:
nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -1;
}

static int cttimeout_default_get(struct sk_buff *skb,
                                 const struct nfnl_info *info,
                                 const struct nlattr * const cda[])
{
        const struct nf_conntrack_l4proto *l4proto;
        unsigned int *timeouts = NULL;
        struct sk_buff *skb2;
        __u16 l3num;
        __u8 l4num;
        int ret;

        if (!cda[CTA_TIMEOUT_L3PROTO] || !cda[CTA_TIMEOUT_L4PROTO])
                return -EINVAL;

        l3num = ntohs(nla_get_be16(cda[CTA_TIMEOUT_L3PROTO]));
        l4num = nla_get_u8(cda[CTA_TIMEOUT_L4PROTO]);
        l4proto = nf_ct_l4proto_find(l4num);

        if (l4proto->l4proto != l4num)
                return -EOPNOTSUPP;

        switch (l4proto->l4proto) {
        case IPPROTO_ICMP:
                timeouts = &nf_icmp_pernet(info->net)->timeout;
                break;
        case IPPROTO_TCP:
                timeouts = nf_tcp_pernet(info->net)->timeouts;
                break;
        case IPPROTO_UDP:
        case IPPROTO_UDPLITE:
                timeouts = nf_udp_pernet(info->net)->timeouts;
                break;
        case IPPROTO_DCCP:
#ifdef CONFIG_NF_CT_PROTO_DCCP
                timeouts = nf_dccp_pernet(info->net)->dccp_timeout;
#endif
                break;
        case IPPROTO_ICMPV6:
                timeouts = &nf_icmpv6_pernet(info->net)->timeout;
                break;
        case IPPROTO_SCTP:
#ifdef CONFIG_NF_CT_PROTO_SCTP
                timeouts = nf_sctp_pernet(info->net)->timeouts;
#endif
                break;
        case IPPROTO_GRE:
#ifdef CONFIG_NF_CT_PROTO_GRE
                timeouts = nf_gre_pernet(info->net)->timeouts;
#endif
                break;
        case 255:
                timeouts = &nf_generic_pernet(info->net)->timeout;
                break;
        default:
                WARN_ONCE(1, "Missing timeouts for proto %d", l4proto->l4proto);
                break;
        }

        if (!timeouts)
                return -EOPNOTSUPP;

        skb2 = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb2)
                return -ENOMEM;

        ret = cttimeout_default_fill_info(info->net, skb2,
                                          NETLINK_CB(skb).portid,
                                          info->nlh->nlmsg_seq,
                                          NFNL_MSG_TYPE(info->nlh->nlmsg_type),
                                          IPCTNL_MSG_TIMEOUT_DEFAULT_SET,
                                          l3num, l4proto, timeouts);
        if (ret <= 0) {
                kfree_skb(skb2);
                return -ENOMEM;
        }

        return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
}

static struct nf_ct_timeout *ctnl_timeout_find_get(struct net *net,
                                                   const char *name)
{
        struct nfct_timeout_pernet *pernet = nfct_timeout_pernet(net);
        struct ctnl_timeout *timeout, *matching = NULL;

        list_for_each_entry_rcu(timeout, &pernet->nfct_timeout_list, head) {
                if (strncmp(timeout->name, name, CTNL_TIMEOUT_NAME_MAX) != 0)
                        continue;

                if (!refcount_inc_not_zero(&timeout->refcnt))
                        goto err;
                matching = timeout;
                break;
        }
err:
        return matching ? &matching->timeout : NULL;
}

static void ctnl_timeout_put(struct nf_ct_timeout *t)
{
        struct ctnl_timeout *timeout =
                container_of(t, struct ctnl_timeout, timeout);

        if (refcount_dec_and_test(&timeout->refcnt)) {
                kfree_rcu(timeout, rcu_head);
                module_put(THIS_MODULE);
        }
}

static const struct nfnl_callback cttimeout_cb[IPCTNL_MSG_TIMEOUT_MAX] = {
        [IPCTNL_MSG_TIMEOUT_NEW] = {
                .call                = cttimeout_new_timeout,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = CTA_TIMEOUT_MAX,
                .policy                = cttimeout_nla_policy
        },
        [IPCTNL_MSG_TIMEOUT_GET] = {
                .call                = cttimeout_get_timeout,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = CTA_TIMEOUT_MAX,
                .policy                = cttimeout_nla_policy
        },
        [IPCTNL_MSG_TIMEOUT_DELETE] = {
                .call                = cttimeout_del_timeout,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = CTA_TIMEOUT_MAX,
                .policy                = cttimeout_nla_policy
        },
        [IPCTNL_MSG_TIMEOUT_DEFAULT_SET] = {
                .call                = cttimeout_default_set,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = CTA_TIMEOUT_MAX,
                .policy                = cttimeout_nla_policy
        },
        [IPCTNL_MSG_TIMEOUT_DEFAULT_GET] = {
                .call                = cttimeout_default_get,
                .type                = NFNL_CB_MUTEX,
                .attr_count        = CTA_TIMEOUT_MAX,
                .policy                = cttimeout_nla_policy
        },
};

static const struct nfnetlink_subsystem cttimeout_subsys = {
        .name                                = "conntrack_timeout",
        .subsys_id                        = NFNL_SUBSYS_CTNETLINK_TIMEOUT,
        .cb_count                        = IPCTNL_MSG_TIMEOUT_MAX,
        .cb                                = cttimeout_cb,
};

MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_CTNETLINK_TIMEOUT);

static int __net_init cttimeout_net_init(struct net *net)
{
        struct nfct_timeout_pernet *pernet = nfct_timeout_pernet(net);

        INIT_LIST_HEAD(&pernet->nfct_timeout_list);
        INIT_LIST_HEAD(&pernet->nfct_timeout_freelist);

        return 0;
}

static void __net_exit cttimeout_net_pre_exit(struct net *net)
{
        struct nfct_timeout_pernet *pernet = nfct_timeout_pernet(net);
        struct ctnl_timeout *cur, *tmp;

        list_for_each_entry_safe(cur, tmp, &pernet->nfct_timeout_list, head) {
                list_del_rcu(&cur->head);
                list_add(&cur->free_head, &pernet->nfct_timeout_freelist);
        }

        /* core calls synchronize_rcu() after this */
}

static void __net_exit cttimeout_net_exit(struct net *net)
{
        struct nfct_timeout_pernet *pernet = nfct_timeout_pernet(net);
        struct ctnl_timeout *cur, *tmp;

        if (list_empty(&pernet->nfct_timeout_freelist))
                return;

        nf_ct_untimeout(net, NULL);

        list_for_each_entry_safe(cur, tmp, &pernet->nfct_timeout_freelist, free_head) {
                list_del(&cur->free_head);

                if (refcount_dec_and_test(&cur->refcnt))
                        kfree_rcu(cur, rcu_head);
        }
}

static struct pernet_operations cttimeout_ops = {
        .init        = cttimeout_net_init,
        .pre_exit = cttimeout_net_pre_exit,
        .exit        = cttimeout_net_exit,
        .id     = &nfct_timeout_id,
        .size   = sizeof(struct nfct_timeout_pernet),
};

static const struct nf_ct_timeout_hooks hooks = {
        .timeout_find_get = ctnl_timeout_find_get,
        .timeout_put = ctnl_timeout_put,
};

static int __init cttimeout_init(void)
{
        int ret;

        ret = register_pernet_subsys(&cttimeout_ops);
        if (ret < 0)
                return ret;

        ret = nfnetlink_subsys_register(&cttimeout_subsys);
        if (ret < 0) {
                pr_err("cttimeout_init: cannot register cttimeout with "
                        "nfnetlink.\n");
                goto err_out;
        }
        RCU_INIT_POINTER(nf_ct_timeout_hook, &hooks);
        return 0;

err_out:
        unregister_pernet_subsys(&cttimeout_ops);
        return ret;
}

static int untimeout(struct nf_conn *ct, void *timeout)
{
        struct nf_conn_timeout *timeout_ext = nf_ct_timeout_find(ct);

        if (timeout_ext)
                RCU_INIT_POINTER(timeout_ext->timeout, NULL);

        return 0;
}

static void __exit cttimeout_exit(void)
{
        nfnetlink_subsys_unregister(&cttimeout_subsys);

        unregister_pernet_subsys(&cttimeout_ops);
        RCU_INIT_POINTER(nf_ct_timeout_hook, NULL);

        nf_ct_iterate_destroy(untimeout, NULL);
}

module_init(cttimeout_init);
module_exit(cttimeout_exit);










































































































































































































  103 
   13 

  110 
   11 






















































    2 
























   92 






  103 









   96 


   37 






   12 







    4 














    3 








    2 







    6 






   26 


   12 













1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * the_nilfs shared structure.
 *
 * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation.
 *
 * Written by Ryusuke Konishi.
 *
 */

#ifndef _THE_NILFS_H
#define _THE_NILFS_H

#include <linux/types.h>
#include <linux/buffer_head.h>
#include <linux/rbtree.h>
#include <linux/fs.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/slab.h>
#include <linux/refcount.h>

struct nilfs_sc_info;
struct nilfs_sysfs_dev_subgroups;

/* the_nilfs struct */
enum {
        THE_NILFS_INIT = 0,     /* Information from super_block is set */
        THE_NILFS_DISCONTINUED,        /* 'next' pointer chain has broken */
        THE_NILFS_GC_RUNNING,        /* gc process is running */
        THE_NILFS_SB_DIRTY,        /* super block is dirty */
        THE_NILFS_PURGING,        /* disposing dirty files for cleanup */
};

/**
 * struct the_nilfs - struct to supervise multiple nilfs mount points
 * @ns_flags: flags
 * @ns_flushed_device: flag indicating if all volatile data was flushed
 * @ns_sb: back pointer to super block instance
 * @ns_bdev: block device
 * @ns_sem: semaphore for shared states
 * @ns_snapshot_mount_mutex: mutex to protect snapshot mounts
 * @ns_sbh: buffer heads of on-disk super blocks
 * @ns_sbp: pointers to super block data
 * @ns_sbwtime: previous write time of super block
 * @ns_sbwcount: write count of super block
 * @ns_sbsize: size of valid data in super block
 * @ns_mount_state: file system state
 * @ns_sb_update_freq: interval of periodical update of superblocks (in seconds)
 * @ns_seg_seq: segment sequence counter
 * @ns_segnum: index number of the latest full segment.
 * @ns_nextnum: index number of the full segment index to be used next
 * @ns_pseg_offset: offset of next partial segment in the current full segment
 * @ns_cno: next checkpoint number
 * @ns_ctime: write time of the last segment
 * @ns_nongc_ctime: write time of the last segment not for cleaner operation
 * @ns_ndirtyblks: Number of dirty data blocks
 * @ns_last_segment_lock: lock protecting fields for the latest segment
 * @ns_last_pseg: start block number of the latest segment
 * @ns_last_seq: sequence value of the latest segment
 * @ns_last_cno: checkpoint number of the latest segment
 * @ns_prot_seq: least sequence number of segments which must not be reclaimed
 * @ns_prev_seq: base sequence number used to decide if advance log cursor
 * @ns_writer: log writer
 * @ns_segctor_sem: semaphore protecting log write
 * @ns_dat: DAT file inode
 * @ns_cpfile: checkpoint file inode
 * @ns_sufile: segusage file inode
 * @ns_cptree: rb-tree of all mounted checkpoints (nilfs_root)
 * @ns_cptree_lock: lock protecting @ns_cptree
 * @ns_dirty_files: list of dirty files
 * @ns_inode_lock: lock protecting @ns_dirty_files
 * @ns_gc_inodes: dummy inodes to keep live blocks
 * @ns_mount_opt: mount options
 * @ns_resuid: uid for reserved blocks
 * @ns_resgid: gid for reserved blocks
 * @ns_interval: checkpoint creation interval
 * @ns_watermark: watermark for the number of dirty buffers
 * @ns_blocksize_bits: bit length of block size
 * @ns_blocksize: block size
 * @ns_nsegments: number of segments in filesystem
 * @ns_blocks_per_segment: number of blocks per segment
 * @ns_r_segments_percentage: reserved segments percentage
 * @ns_nrsvsegs: number of reserved segments
 * @ns_first_data_block: block number of first data block
 * @ns_inode_size: size of on-disk inode
 * @ns_first_ino: first not-special inode number
 * @ns_crc_seed: seed value of CRC32 calculation
 * @ns_dev_kobj: /sys/fs/<nilfs>/<device>
 * @ns_dev_kobj_unregister: completion state
 * @ns_dev_subgroups: <device> subgroups pointer
 */
struct the_nilfs {
        unsigned long                ns_flags;
        int                        ns_flushed_device;

        struct super_block     *ns_sb;
        struct block_device    *ns_bdev;
        struct rw_semaphore        ns_sem;
        struct mutex                ns_snapshot_mount_mutex;

        /*
         * used for
         * - loading the latest checkpoint exclusively.
         * - allocating a new full segment.
         */
        struct buffer_head     *ns_sbh[2];
        struct nilfs_super_block *ns_sbp[2];
        time64_t                ns_sbwtime;
        unsigned int                ns_sbwcount;
        unsigned int                ns_sbsize;
        unsigned int                ns_mount_state;
        unsigned int                ns_sb_update_freq;

        /*
         * The following fields are updated by a writable FS-instance.
         * These fields are protected by ns_segctor_sem outside load_nilfs().
         */
        u64                        ns_seg_seq;
        __u64                        ns_segnum;
        __u64                        ns_nextnum;
        unsigned long                ns_pseg_offset;
        __u64                        ns_cno;
        time64_t                ns_ctime;
        time64_t                ns_nongc_ctime;
        atomic_t                ns_ndirtyblks;

        /*
         * The following fields hold information on the latest partial segment
         * written to disk with a super root.  These fields are protected by
         * ns_last_segment_lock.
         */
        spinlock_t                ns_last_segment_lock;
        sector_t                ns_last_pseg;
        u64                        ns_last_seq;
        __u64                        ns_last_cno;
        u64                        ns_prot_seq;
        u64                        ns_prev_seq;

        struct nilfs_sc_info   *ns_writer;
        struct rw_semaphore        ns_segctor_sem;

        /*
         * Following fields are lock free except for the period before
         * the_nilfs is initialized.
         */
        struct inode               *ns_dat;
        struct inode               *ns_cpfile;
        struct inode               *ns_sufile;

        /* Checkpoint tree */
        struct rb_root                ns_cptree;
        spinlock_t                ns_cptree_lock;

        /* Dirty inode list */
        struct list_head        ns_dirty_files;
        spinlock_t                ns_inode_lock;

        /* GC inode list */
        struct list_head        ns_gc_inodes;

        /* Mount options */
        unsigned long                ns_mount_opt;

        uid_t                        ns_resuid;
        gid_t                        ns_resgid;
        unsigned long                ns_interval;
        unsigned long                ns_watermark;

        /* Disk layout information (static) */
        unsigned int                ns_blocksize_bits;
        unsigned int                ns_blocksize;
        unsigned long                ns_nsegments;
        unsigned long                ns_blocks_per_segment;
        unsigned long                ns_r_segments_percentage;
        unsigned long                ns_nrsvsegs;
        unsigned long                ns_first_data_block;
        int                        ns_inode_size;
        unsigned int                ns_first_ino;
        u32                        ns_crc_seed;

        /* /sys/fs/<nilfs>/<device> */
        struct kobject ns_dev_kobj;
        struct completion ns_dev_kobj_unregister;
        struct nilfs_sysfs_dev_subgroups *ns_dev_subgroups;
};

#define THE_NILFS_FNS(bit, name)                                        \
static inline void set_nilfs_##name(struct the_nilfs *nilfs)                \
{                                                                        \
        set_bit(THE_NILFS_##bit, &(nilfs)->ns_flags);                        \
}                                                                        \
static inline void clear_nilfs_##name(struct the_nilfs *nilfs)                \
{                                                                        \
        clear_bit(THE_NILFS_##bit, &(nilfs)->ns_flags);                        \
}                                                                        \
static inline int nilfs_##name(struct the_nilfs *nilfs)                        \
{                                                                        \
        return test_bit(THE_NILFS_##bit, &(nilfs)->ns_flags);                \
}

THE_NILFS_FNS(INIT, init)
THE_NILFS_FNS(DISCONTINUED, discontinued)
THE_NILFS_FNS(GC_RUNNING, gc_running)
THE_NILFS_FNS(SB_DIRTY, sb_dirty)
THE_NILFS_FNS(PURGING, purging)

/*
 * Mount option operations
 */
#define nilfs_clear_opt(nilfs, opt)  \
        ((nilfs)->ns_mount_opt &= ~NILFS_MOUNT_##opt)
#define nilfs_set_opt(nilfs, opt)  \
        ((nilfs)->ns_mount_opt |= NILFS_MOUNT_##opt)
#define nilfs_test_opt(nilfs, opt) ((nilfs)->ns_mount_opt & NILFS_MOUNT_##opt)

/**
 * struct nilfs_root - nilfs root object
 * @cno: checkpoint number
 * @rb_node: red-black tree node
 * @count: refcount of this structure
 * @nilfs: nilfs object
 * @ifile: inode file
 * @inodes_count: number of inodes
 * @blocks_count: number of blocks
 * @snapshot_kobj: /sys/fs/<nilfs>/<device>/mounted_snapshots/<snapshot>
 * @snapshot_kobj_unregister: completion state for kernel object
 */
struct nilfs_root {
        __u64 cno;
        struct rb_node rb_node;

        refcount_t count;
        struct the_nilfs *nilfs;
        struct inode *ifile;

        atomic64_t inodes_count;
        atomic64_t blocks_count;

        /* /sys/fs/<nilfs>/<device>/mounted_snapshots/<snapshot> */
        struct kobject snapshot_kobj;
        struct completion snapshot_kobj_unregister;
};

/* Special checkpoint number */
#define NILFS_CPTREE_CURRENT_CNO        0

/* Minimum interval of periodical update of superblocks (in seconds) */
#define NILFS_SB_FREQ                10

static inline int nilfs_sb_need_update(struct the_nilfs *nilfs)
{
        u64 t = ktime_get_real_seconds();

        return t < nilfs->ns_sbwtime ||
                t > nilfs->ns_sbwtime + nilfs->ns_sb_update_freq;
}

static inline int nilfs_sb_will_flip(struct the_nilfs *nilfs)
{
        int flip_bits = nilfs->ns_sbwcount & 0x0FL;

        return (flip_bits != 0x08 && flip_bits != 0x0F);
}

void nilfs_set_last_segment(struct the_nilfs *, sector_t, u64, __u64);
struct the_nilfs *alloc_nilfs(struct super_block *sb);
void destroy_nilfs(struct the_nilfs *nilfs);
int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb);
int load_nilfs(struct the_nilfs *nilfs, struct super_block *sb);
unsigned long nilfs_nrsvsegs(struct the_nilfs *nilfs, unsigned long nsegs);
void nilfs_set_nsegments(struct the_nilfs *nilfs, unsigned long nsegs);
int nilfs_discard_segments(struct the_nilfs *, __u64 *, size_t);
int nilfs_count_free_blocks(struct the_nilfs *, sector_t *);
struct nilfs_root *nilfs_lookup_root(struct the_nilfs *nilfs, __u64 cno);
struct nilfs_root *nilfs_find_or_create_root(struct the_nilfs *nilfs,
                                             __u64 cno);
void nilfs_put_root(struct nilfs_root *root);
int nilfs_near_disk_full(struct the_nilfs *);
void nilfs_fall_back_super_block(struct the_nilfs *);
void nilfs_swap_super_block(struct the_nilfs *);


static inline void nilfs_get_root(struct nilfs_root *root)
{
        refcount_inc(&root->count);
}

static inline int nilfs_valid_fs(struct the_nilfs *nilfs)
{
        unsigned int valid_fs;

        down_read(&nilfs->ns_sem);
        valid_fs = (nilfs->ns_mount_state & NILFS_VALID_FS);
        up_read(&nilfs->ns_sem);
        return valid_fs;
}

static inline void
nilfs_get_segment_range(struct the_nilfs *nilfs, __u64 segnum,
                        sector_t *seg_start, sector_t *seg_end)
{
        *seg_start = (sector_t)nilfs->ns_blocks_per_segment * segnum;
        *seg_end = *seg_start + nilfs->ns_blocks_per_segment - 1;
        if (segnum == 0)
                *seg_start = nilfs->ns_first_data_block;
}

static inline sector_t
nilfs_get_segment_start_blocknr(struct the_nilfs *nilfs, __u64 segnum)
{
        return (segnum == 0) ? nilfs->ns_first_data_block :
                (sector_t)nilfs->ns_blocks_per_segment * segnum;
}

static inline __u64
nilfs_get_segnum_of_block(struct the_nilfs *nilfs, sector_t blocknr)
{
        sector_t segnum = blocknr;

        sector_div(segnum, nilfs->ns_blocks_per_segment);
        return segnum;
}

static inline void
nilfs_terminate_segment(struct the_nilfs *nilfs, sector_t seg_start,
                        sector_t seg_end)
{
        /* terminate the current full segment (used in case of I/O-error) */
        nilfs->ns_pseg_offset = seg_end - seg_start + 1;
}

static inline void nilfs_shift_to_next_segment(struct the_nilfs *nilfs)
{
        /* move forward with a full segment */
        nilfs->ns_segnum = nilfs->ns_nextnum;
        nilfs->ns_pseg_offset = 0;
        nilfs->ns_seg_seq++;
}

static inline __u64 nilfs_last_cno(struct the_nilfs *nilfs)
{
        __u64 cno;

        spin_lock(&nilfs->ns_last_segment_lock);
        cno = nilfs->ns_last_cno;
        spin_unlock(&nilfs->ns_last_segment_lock);
        return cno;
}

static inline int nilfs_segment_is_active(struct the_nilfs *nilfs, __u64 n)
{
        return n == nilfs->ns_segnum || n == nilfs->ns_nextnum;
}

static inline int nilfs_flush_device(struct the_nilfs *nilfs)
{
        int err;

        if (!nilfs_test_opt(nilfs, BARRIER) || nilfs->ns_flushed_device)
                return 0;

        nilfs->ns_flushed_device = 1;
        /*
         * the store to ns_flushed_device must not be reordered after
         * blkdev_issue_flush().
         */
        smp_wmb();

        err = blkdev_issue_flush(nilfs->ns_bdev);
        if (err != -EIO)
                err = 0;
        return err;
}

#endif /* _THE_NILFS_H */
















































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MIGRATE_H
#define _LINUX_MIGRATE_H

#include <linux/mm.h>
#include <linux/mempolicy.h>
#include <linux/migrate_mode.h>
#include <linux/hugetlb.h>

typedef struct folio *new_folio_t(struct folio *folio, unsigned long private);
typedef void free_folio_t(struct folio *folio, unsigned long private);

struct migration_target_control;

/*
 * Return values from addresss_space_operations.migratepage():
 * - negative errno on page migration failure;
 * - zero on page migration success;
 */
#define MIGRATEPAGE_SUCCESS                0
#define MIGRATEPAGE_UNMAP                1

/**
 * struct movable_operations - Driver page migration
 * @isolate_page:
 * The VM calls this function to prepare the page to be moved.  The page
 * is locked and the driver should not unlock it.  The driver should
 * return ``true`` if the page is movable and ``false`` if it is not
 * currently movable.  After this function returns, the VM uses the
 * page->lru field, so the driver must preserve any information which
 * is usually stored here.
 *
 * @migrate_page:
 * After isolation, the VM calls this function with the isolated
 * @src page.  The driver should copy the contents of the
 * @src page to the @dst page and set up the fields of @dst page.
 * Both pages are locked.
 * If page migration is successful, the driver should call
 * __ClearPageMovable(@src) and return MIGRATEPAGE_SUCCESS.
 * If the driver cannot migrate the page at the moment, it can return
 * -EAGAIN.  The VM interprets this as a temporary migration failure and
 * will retry it later.  Any other error value is a permanent migration
 * failure and migration will not be retried.
 * The driver shouldn't touch the @src->lru field while in the
 * migrate_page() function.  It may write to @dst->lru.
 *
 * @putback_page:
 * If migration fails on the isolated page, the VM informs the driver
 * that the page is no longer a candidate for migration by calling
 * this function.  The driver should put the isolated page back into
 * its own data structure.
 */
struct movable_operations {
        bool (*isolate_page)(struct page *, isolate_mode_t);
        int (*migrate_page)(struct page *dst, struct page *src,
                        enum migrate_mode);
        void (*putback_page)(struct page *);
};

/* Defined in mm/debug.c: */
extern const char *migrate_reason_names[MR_TYPES];

#ifdef CONFIG_MIGRATION

void putback_movable_pages(struct list_head *l);
int migrate_folio(struct address_space *mapping, struct folio *dst,
                struct folio *src, enum migrate_mode mode);
int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free,
                  unsigned long private, enum migrate_mode mode, int reason,
                  unsigned int *ret_succeeded);
struct folio *alloc_migration_target(struct folio *src, unsigned long private);
bool isolate_movable_page(struct page *page, isolate_mode_t mode);
bool isolate_folio_to_list(struct folio *folio, struct list_head *list);

int migrate_huge_page_move_mapping(struct address_space *mapping,
                struct folio *dst, struct folio *src);
void migration_entry_wait_on_locked(swp_entry_t entry, spinlock_t *ptl)
                __releases(ptl);
void folio_migrate_flags(struct folio *newfolio, struct folio *folio);
int folio_migrate_mapping(struct address_space *mapping,
                struct folio *newfolio, struct folio *folio, int extra_count);

#else

static inline void putback_movable_pages(struct list_head *l) {}
static inline int migrate_pages(struct list_head *l, new_folio_t new,
                free_folio_t free, unsigned long private,
                enum migrate_mode mode, int reason, unsigned int *ret_succeeded)
        { return -ENOSYS; }
static inline struct folio *alloc_migration_target(struct folio *src,
                unsigned long private)
        { return NULL; }
static inline bool isolate_movable_page(struct page *page, isolate_mode_t mode)
        { return false; }
static inline bool isolate_folio_to_list(struct folio *folio, struct list_head *list)
        { return false; }

static inline int migrate_huge_page_move_mapping(struct address_space *mapping,
                                  struct folio *dst, struct folio *src)
{
        return -ENOSYS;
}

#endif /* CONFIG_MIGRATION */

#ifdef CONFIG_COMPACTION
bool PageMovable(struct page *page);
void __SetPageMovable(struct page *page, const struct movable_operations *ops);
void __ClearPageMovable(struct page *page);
#else
static inline bool PageMovable(struct page *page) { return false; }
static inline void __SetPageMovable(struct page *page,
                const struct movable_operations *ops)
{
}
static inline void __ClearPageMovable(struct page *page)
{
}
#endif

static inline bool folio_test_movable(struct folio *folio)
{
        return PageMovable(&folio->page);
}

static inline
const struct movable_operations *folio_movable_ops(struct folio *folio)
{
        VM_BUG_ON(!__folio_test_movable(folio));

        return (const struct movable_operations *)
                ((unsigned long)folio->mapping - PAGE_MAPPING_MOVABLE);
}

static inline
const struct movable_operations *page_movable_ops(struct page *page)
{
        VM_BUG_ON(!__PageMovable(page));

        return (const struct movable_operations *)
                ((unsigned long)page->mapping - PAGE_MAPPING_MOVABLE);
}

#ifdef CONFIG_NUMA_BALANCING
int migrate_misplaced_folio_prepare(struct folio *folio,
                struct vm_area_struct *vma, int node);
int migrate_misplaced_folio(struct folio *folio, struct vm_area_struct *vma,
                           int node);
#else
static inline int migrate_misplaced_folio_prepare(struct folio *folio,
                struct vm_area_struct *vma, int node)
{
        return -EAGAIN; /* can't migrate now */
}
static inline int migrate_misplaced_folio(struct folio *folio,
                                         struct vm_area_struct *vma, int node)
{
        return -EAGAIN; /* can't migrate now */
}
#endif /* CONFIG_NUMA_BALANCING */

#ifdef CONFIG_MIGRATION

/*
 * Watch out for PAE architecture, which has an unsigned long, and might not
 * have enough bits to store all physical address and flags. So far we have
 * enough room for all our flags.
 */
#define MIGRATE_PFN_VALID        (1UL << 0)
#define MIGRATE_PFN_MIGRATE        (1UL << 1)
#define MIGRATE_PFN_WRITE        (1UL << 3)
#define MIGRATE_PFN_SHIFT        6

static inline struct page *migrate_pfn_to_page(unsigned long mpfn)
{
        if (!(mpfn & MIGRATE_PFN_VALID))
                return NULL;
        return pfn_to_page(mpfn >> MIGRATE_PFN_SHIFT);
}

static inline unsigned long migrate_pfn(unsigned long pfn)
{
        return (pfn << MIGRATE_PFN_SHIFT) | MIGRATE_PFN_VALID;
}

enum migrate_vma_direction {
        MIGRATE_VMA_SELECT_SYSTEM = 1 << 0,
        MIGRATE_VMA_SELECT_DEVICE_PRIVATE = 1 << 1,
        MIGRATE_VMA_SELECT_DEVICE_COHERENT = 1 << 2,
};

struct migrate_vma {
        struct vm_area_struct        *vma;
        /*
         * Both src and dst array must be big enough for
         * (end - start) >> PAGE_SHIFT entries.
         *
         * The src array must not be modified by the caller after
         * migrate_vma_setup(), and must not change the dst array after
         * migrate_vma_pages() returns.
         */
        unsigned long                *dst;
        unsigned long                *src;
        unsigned long                cpages;
        unsigned long                npages;
        unsigned long                start;
        unsigned long                end;

        /*
         * Set to the owner value also stored in page->pgmap->owner for
         * migrating out of device private memory. The flags also need to
         * be set to MIGRATE_VMA_SELECT_DEVICE_PRIVATE.
         * The caller should always set this field when using mmu notifier
         * callbacks to avoid device MMU invalidations for device private
         * pages that are not being migrated.
         */
        void                        *pgmap_owner;
        unsigned long                flags;

        /*
         * Set to vmf->page if this is being called to migrate a page as part of
         * a migrate_to_ram() callback.
         */
        struct page                *fault_page;
};

int migrate_vma_setup(struct migrate_vma *args);
void migrate_vma_pages(struct migrate_vma *migrate);
void migrate_vma_finalize(struct migrate_vma *migrate);
int migrate_device_range(unsigned long *src_pfns, unsigned long start,
                        unsigned long npages);
void migrate_device_pages(unsigned long *src_pfns, unsigned long *dst_pfns,
                        unsigned long npages);
void migrate_device_finalize(unsigned long *src_pfns,
                        unsigned long *dst_pfns, unsigned long npages);

#endif /* CONFIG_MIGRATION */

#endif /* _LINUX_MIGRATE_H */







































































































































































































































































































































































































































































































































































































































































































 6164 
   63 



 6187 

   63 














  895 


























 9795 




 9787 

 9801 














 8382 




 8384 

 8376 













  417 












   68 













  205 













   41 













   61 
   61 





   60 




























 3178 



























    6 






    4 


    3 




    3 



    3 

    3 



    6 

















































































   24 














   54 














   14 















    5 


















   54 




































    3 




















   21 





















 18935 















  105 












    2 














   25 














    7 



























 1197 
 1195 





 1193 


















   94 























    1 
















   86 






























































  143 





















 3409 
 3409 


 3400 

















 3178 



 3183 














  140 











  106 
  105 











 3611 

















 3497 

































  229 













 2171 













  126 













  307 






















 3809 













   39 














   14 



















 2198 

 2199 
 2196 




 2197 




































   26 
















   76 















 8405 



 8408 






























 4569 
 4559 

 4558 






















    3 























   43 





































 1133 


 1135 

 1134 


 1134 

 1116 





 1135 
 1135 

 1136 










 1135 






 1133 




















  352 



















 1153 
 1154 
 1155 












   67 

   67 















  556 
  555 
  556 














  107 
  107 
  107 













  158 
  157 
  158 
















  166 
  167 
  167 















   91 
   91 
   91 


















  366 


  364 

  366 
















  156 
  157 
  156 















   68 
   69 
   66 














  105 
  105 
  104 












    5 
















 1210 
 1207 
 1210 













  185 

  185 















  118 
  116 
  118 













  310 
  310 
  311 















  165 
  165 
  165 















  670 
  668 
  670 














  106 
  106 
  106 



















  111 
  112 
  112 


















  513 




  513 
   61 





  513 













   30 
   30 
   30 
















  928 
  930 
  932 


















 9515 
 9514 
 9528 


















  799 
  799 
  790 














  706 

  698 












  199 
  201 
  199 


































  351 
  351 


  351 
  350 




  350 



















   62 
   63 
   62 














   37 

   37 
















   12 
   12 
   11 
















  175 
  175 
  174 














  162 

  162 















  299 

  299 














  310 
  309 
  309 













  162 
  162 
  161 






























  193 
  192 


  192 
  190 




  191 











  168 

  168 















 1351 
















    6 






















   81 


   81 





















    2 


    2 



















    9 
    9 
    8 





























   72 




















   10 








































 4359 























 7049 













 9816 



 9816 













 3684 












 5632 

 5615 

 5618 



















 9325 



























 1311 

  760 




    3 
    2 




    2 



























 1311 













 1474 















   89 














   84 


















  232 















   40 



















   54 













   21 













 5967 















 5491 















  241 













  417 



  418 














   58 

   58 















    9 




    9 

















    5 


    5 

    5 















  895 




  899 



























    1 
    1 













   83 
   83 













































  544 















   21 



















   16 














   42 





















    2 




















   67 



















   41 
















   10 














    8 













    1 
























 6558 
 6567 





























   17 













   90 












   18 
















    4 

















   57 













   93 












   13 












   16 




















   49 























  302 
  292 


   33 



  291 












  615 












    9 













  119 



























   41 



   42 













   48 
   47 














   20 



   21 














































   31 
















   18 




















   15 













   31 



   31 
































    1 














   37 

















   25 













   18 



   18 































    1 














   54 

















   34 











 9895 

 7987 



























   17 










   16 
   16 

   14 


   13 



    3 

    3 




    3 








   12 
   12 

   11 

    3 
   11 




    8 



    6 


    5 

    5 




    9 

    8 

    6 






























   12 

   12 

    9 


    8 

    2 

    6 
    6 





    4 
    4 
    3 




    6 
   12 


















    5 
    5 

    5 




















   26 
   27 

   27 



















 7990 































   24 




















  398 















   14 
   14 












  398 












    3 














































































   13 















   12 




























    9 

























  108 
















 1731 























 1618 















  171 


















  801 
















  922 













  183 















  199 














 11030 
















 1334 













   55 












   45 















 1273 














 3095 














  169 
















 10678 





















    1 




















 8259 














 2114 















 2114 



 2123 













  824 
  819 












   44 












 1123 













   85 













  109 
















   79 


























   70 













    2 










    2 










    2 












   63 



   63 



   63 
















   17 












   65 













    2 















   71 














    8 















  110 




















  310 
















   24 
















    1 

















  142 




























































































    8 
















    2 










   17 













    5 
















    5 


















   22 












    3 




























  123 




























    1 
    1 


    1 













  225 




  218 






















  205 



  208 






























  458 

















    3 
    3 

















   97 





















































































 4179 














  237 













 1124 
















  235 

















 1677 






































































  540 

























  253 

















   61 



   61 























































































    2 




    2 


























































    3 












   41 












   22 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Security plug functions
 *
 * Copyright (C) 2001 WireX Communications, Inc <chris@wirex.com>
 * Copyright (C) 2001-2002 Greg Kroah-Hartman <greg@kroah.com>
 * Copyright (C) 2001 Networks Associates Technology, Inc <ssmalley@nai.com>
 * Copyright (C) 2016 Mellanox Technologies
 * Copyright (C) 2023 Microsoft Corporation <paul@paul-moore.com>
 */

#define pr_fmt(fmt) "LSM: " fmt

#include <linux/bpf.h>
#include <linux/capability.h>
#include <linux/dcache.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/kernel_read_file.h>
#include <linux/lsm_hooks.h>
#include <linux/mman.h>
#include <linux/mount.h>
#include <linux/personality.h>
#include <linux/backing-dev.h>
#include <linux/string.h>
#include <linux/xattr.h>
#include <linux/msg.h>
#include <linux/overflow.h>
#include <linux/perf_event.h>
#include <linux/fs.h>
#include <net/flow.h>
#include <net/sock.h>

#define SECURITY_HOOK_ACTIVE_KEY(HOOK, IDX) security_hook_active_##HOOK##_##IDX

/*
 * Identifier for the LSM static calls.
 * HOOK is an LSM hook as defined in linux/lsm_hookdefs.h
 * IDX is the index of the static call. 0 <= NUM < MAX_LSM_COUNT
 */
#define LSM_STATIC_CALL(HOOK, IDX) lsm_static_call_##HOOK##_##IDX

/*
 * Call the macro M for each LSM hook MAX_LSM_COUNT times.
 */
#define LSM_LOOP_UNROLL(M, ...)                 \
do {                                                \
        UNROLL(MAX_LSM_COUNT, M, __VA_ARGS__)        \
} while (0)

#define LSM_DEFINE_UNROLL(M, ...) UNROLL(MAX_LSM_COUNT, M, __VA_ARGS__)

/*
 * These are descriptions of the reasons that can be passed to the
 * security_locked_down() LSM hook. Placing this array here allows
 * all security modules to use the same descriptions for auditing
 * purposes.
 */
const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX + 1] = {
        [LOCKDOWN_NONE] = "none",
        [LOCKDOWN_MODULE_SIGNATURE] = "unsigned module loading",
        [LOCKDOWN_DEV_MEM] = "/dev/mem,kmem,port",
        [LOCKDOWN_EFI_TEST] = "/dev/efi_test access",
        [LOCKDOWN_KEXEC] = "kexec of unsigned images",
        [LOCKDOWN_HIBERNATION] = "hibernation",
        [LOCKDOWN_PCI_ACCESS] = "direct PCI access",
        [LOCKDOWN_IOPORT] = "raw io port access",
        [LOCKDOWN_MSR] = "raw MSR access",
        [LOCKDOWN_ACPI_TABLES] = "modifying ACPI tables",
        [LOCKDOWN_DEVICE_TREE] = "modifying device tree contents",
        [LOCKDOWN_PCMCIA_CIS] = "direct PCMCIA CIS storage",
        [LOCKDOWN_TIOCSSERIAL] = "reconfiguration of serial port IO",
        [LOCKDOWN_MODULE_PARAMETERS] = "unsafe module parameters",
        [LOCKDOWN_MMIOTRACE] = "unsafe mmio",
        [LOCKDOWN_DEBUGFS] = "debugfs access",
        [LOCKDOWN_XMON_WR] = "xmon write access",
        [LOCKDOWN_BPF_WRITE_USER] = "use of bpf to write user RAM",
        [LOCKDOWN_DBG_WRITE_KERNEL] = "use of kgdb/kdb to write kernel RAM",
        [LOCKDOWN_RTAS_ERROR_INJECTION] = "RTAS error injection",
        [LOCKDOWN_INTEGRITY_MAX] = "integrity",
        [LOCKDOWN_KCORE] = "/proc/kcore access",
        [LOCKDOWN_KPROBES] = "use of kprobes",
        [LOCKDOWN_BPF_READ_KERNEL] = "use of bpf to read kernel RAM",
        [LOCKDOWN_DBG_READ_KERNEL] = "use of kgdb/kdb to read kernel RAM",
        [LOCKDOWN_PERF] = "unsafe use of perf",
        [LOCKDOWN_TRACEFS] = "use of tracefs",
        [LOCKDOWN_XMON_RW] = "xmon read and write access",
        [LOCKDOWN_XFRM_SECRET] = "xfrm SA secret",
        [LOCKDOWN_CONFIDENTIALITY_MAX] = "confidentiality",
};

static BLOCKING_NOTIFIER_HEAD(blocking_lsm_notifier_chain);

static struct kmem_cache *lsm_file_cache;
static struct kmem_cache *lsm_inode_cache;

char *lsm_names;
static struct lsm_blob_sizes blob_sizes __ro_after_init;

/* Boot-time LSM user choice */
static __initdata const char *chosen_lsm_order;
static __initdata const char *chosen_major_lsm;

static __initconst const char *const builtin_lsm_order = CONFIG_LSM;

/* Ordered list of LSMs to initialize. */
static __initdata struct lsm_info *ordered_lsms[MAX_LSM_COUNT + 1];
static __initdata struct lsm_info *exclusive;

#ifdef CONFIG_HAVE_STATIC_CALL
#define LSM_HOOK_TRAMP(NAME, NUM) \
        &STATIC_CALL_TRAMP(LSM_STATIC_CALL(NAME, NUM))
#else
#define LSM_HOOK_TRAMP(NAME, NUM) NULL
#endif

/*
 * Define static calls and static keys for each LSM hook.
 */
#define DEFINE_LSM_STATIC_CALL(NUM, NAME, RET, ...)                        \
        DEFINE_STATIC_CALL_NULL(LSM_STATIC_CALL(NAME, NUM),                \
                                *((RET(*)(__VA_ARGS__))NULL));                \
        DEFINE_STATIC_KEY_FALSE(SECURITY_HOOK_ACTIVE_KEY(NAME, NUM));

#define LSM_HOOK(RET, DEFAULT, NAME, ...)                                \
        LSM_DEFINE_UNROLL(DEFINE_LSM_STATIC_CALL, NAME, RET, __VA_ARGS__)
#include <linux/lsm_hook_defs.h>
#undef LSM_HOOK
#undef DEFINE_LSM_STATIC_CALL

/*
 * Initialise a table of static calls for each LSM hook.
 * DEFINE_STATIC_CALL_NULL invocation above generates a key (STATIC_CALL_KEY)
 * and a trampoline (STATIC_CALL_TRAMP) which are used to call
 * __static_call_update when updating the static call.
 *
 * The static calls table is used by early LSMs, some architectures can fault on
 * unaligned accesses and the fault handling code may not be ready by then.
 * Thus, the static calls table should be aligned to avoid any unhandled faults
 * in early init.
 */
struct lsm_static_calls_table
        static_calls_table __ro_after_init __aligned(sizeof(u64)) = {
#define INIT_LSM_STATIC_CALL(NUM, NAME)                                        \
        (struct lsm_static_call) {                                        \
                .key = &STATIC_CALL_KEY(LSM_STATIC_CALL(NAME, NUM)),        \
                .trampoline = LSM_HOOK_TRAMP(NAME, NUM),                \
                .active = &SECURITY_HOOK_ACTIVE_KEY(NAME, NUM),                \
        },
#define LSM_HOOK(RET, DEFAULT, NAME, ...)                                \
        .NAME = {                                                        \
                LSM_DEFINE_UNROLL(INIT_LSM_STATIC_CALL, NAME)                \
        },
#include <linux/lsm_hook_defs.h>
#undef LSM_HOOK
#undef INIT_LSM_STATIC_CALL
        };

static __initdata bool debug;
#define init_debug(...)                                                \
        do {                                                        \
                if (debug)                                        \
                        pr_info(__VA_ARGS__);                        \
        } while (0)

static bool __init is_enabled(struct lsm_info *lsm)
{
        if (!lsm->enabled)
                return false;

        return *lsm->enabled;
}

/* Mark an LSM's enabled flag. */
static int lsm_enabled_true __initdata = 1;
static int lsm_enabled_false __initdata = 0;
static void __init set_enabled(struct lsm_info *lsm, bool enabled)
{
        /*
         * When an LSM hasn't configured an enable variable, we can use
         * a hard-coded location for storing the default enabled state.
         */
        if (!lsm->enabled) {
                if (enabled)
                        lsm->enabled = &lsm_enabled_true;
                else
                        lsm->enabled = &lsm_enabled_false;
        } else if (lsm->enabled == &lsm_enabled_true) {
                if (!enabled)
                        lsm->enabled = &lsm_enabled_false;
        } else if (lsm->enabled == &lsm_enabled_false) {
                if (enabled)
                        lsm->enabled = &lsm_enabled_true;
        } else {
                *lsm->enabled = enabled;
        }
}

/* Is an LSM already listed in the ordered LSMs list? */
static bool __init exists_ordered_lsm(struct lsm_info *lsm)
{
        struct lsm_info **check;

        for (check = ordered_lsms; *check; check++)
                if (*check == lsm)
                        return true;

        return false;
}

/* Append an LSM to the list of ordered LSMs to initialize. */
static int last_lsm __initdata;
static void __init append_ordered_lsm(struct lsm_info *lsm, const char *from)
{
        /* Ignore duplicate selections. */
        if (exists_ordered_lsm(lsm))
                return;

        if (WARN(last_lsm == MAX_LSM_COUNT, "%s: out of LSM static calls!?\n", from))
                return;

        /* Enable this LSM, if it is not already set. */
        if (!lsm->enabled)
                lsm->enabled = &lsm_enabled_true;
        ordered_lsms[last_lsm++] = lsm;

        init_debug("%s ordered: %s (%s)\n", from, lsm->name,
                   is_enabled(lsm) ? "enabled" : "disabled");
}

/* Is an LSM allowed to be initialized? */
static bool __init lsm_allowed(struct lsm_info *lsm)
{
        /* Skip if the LSM is disabled. */
        if (!is_enabled(lsm))
                return false;

        /* Not allowed if another exclusive LSM already initialized. */
        if ((lsm->flags & LSM_FLAG_EXCLUSIVE) && exclusive) {
                init_debug("exclusive disabled: %s\n", lsm->name);
                return false;
        }

        return true;
}

static void __init lsm_set_blob_size(int *need, int *lbs)
{
        int offset;

        if (*need <= 0)
                return;

        offset = ALIGN(*lbs, sizeof(void *));
        *lbs = offset + *need;
        *need = offset;
}

static void __init lsm_set_blob_sizes(struct lsm_blob_sizes *needed)
{
        if (!needed)
                return;

        lsm_set_blob_size(&needed->lbs_cred, &blob_sizes.lbs_cred);
        lsm_set_blob_size(&needed->lbs_file, &blob_sizes.lbs_file);
        lsm_set_blob_size(&needed->lbs_ib, &blob_sizes.lbs_ib);
        /*
         * The inode blob gets an rcu_head in addition to
         * what the modules might need.
         */
        if (needed->lbs_inode && blob_sizes.lbs_inode == 0)
                blob_sizes.lbs_inode = sizeof(struct rcu_head);
        lsm_set_blob_size(&needed->lbs_inode, &blob_sizes.lbs_inode);
        lsm_set_blob_size(&needed->lbs_ipc, &blob_sizes.lbs_ipc);
        lsm_set_blob_size(&needed->lbs_key, &blob_sizes.lbs_key);
        lsm_set_blob_size(&needed->lbs_msg_msg, &blob_sizes.lbs_msg_msg);
        lsm_set_blob_size(&needed->lbs_perf_event, &blob_sizes.lbs_perf_event);
        lsm_set_blob_size(&needed->lbs_sock, &blob_sizes.lbs_sock);
        lsm_set_blob_size(&needed->lbs_superblock, &blob_sizes.lbs_superblock);
        lsm_set_blob_size(&needed->lbs_task, &blob_sizes.lbs_task);
        lsm_set_blob_size(&needed->lbs_tun_dev, &blob_sizes.lbs_tun_dev);
        lsm_set_blob_size(&needed->lbs_xattr_count,
                          &blob_sizes.lbs_xattr_count);
        lsm_set_blob_size(&needed->lbs_bdev, &blob_sizes.lbs_bdev);
}

/* Prepare LSM for initialization. */
static void __init prepare_lsm(struct lsm_info *lsm)
{
        int enabled = lsm_allowed(lsm);

        /* Record enablement (to handle any following exclusive LSMs). */
        set_enabled(lsm, enabled);

        /* If enabled, do pre-initialization work. */
        if (enabled) {
                if ((lsm->flags & LSM_FLAG_EXCLUSIVE) && !exclusive) {
                        exclusive = lsm;
                        init_debug("exclusive chosen:   %s\n", lsm->name);
                }

                lsm_set_blob_sizes(lsm->blobs);
        }
}

/* Initialize a given LSM, if it is enabled. */
static void __init initialize_lsm(struct lsm_info *lsm)
{
        if (is_enabled(lsm)) {
                int ret;

                init_debug("initializing %s\n", lsm->name);
                ret = lsm->init();
                WARN(ret, "%s failed to initialize: %d\n", lsm->name, ret);
        }
}

/*
 * Current index to use while initializing the lsm id list.
 */
u32 lsm_active_cnt __ro_after_init;
const struct lsm_id *lsm_idlist[MAX_LSM_COUNT];

/* Populate ordered LSMs list from comma-separated LSM name list. */
static void __init ordered_lsm_parse(const char *order, const char *origin)
{
        struct lsm_info *lsm;
        char *sep, *name, *next;

        /* LSM_ORDER_FIRST is always first. */
        for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
                if (lsm->order == LSM_ORDER_FIRST)
                        append_ordered_lsm(lsm, "  first");
        }

        /* Process "security=", if given. */
        if (chosen_major_lsm) {
                struct lsm_info *major;

                /*
                 * To match the original "security=" behavior, this
                 * explicitly does NOT fallback to another Legacy Major
                 * if the selected one was separately disabled: disable
                 * all non-matching Legacy Major LSMs.
                 */
                for (major = __start_lsm_info; major < __end_lsm_info;
                     major++) {
                        if ((major->flags & LSM_FLAG_LEGACY_MAJOR) &&
                            strcmp(major->name, chosen_major_lsm) != 0) {
                                set_enabled(major, false);
                                init_debug("security=%s disabled: %s (only one legacy major LSM)\n",
                                           chosen_major_lsm, major->name);
                        }
                }
        }

        sep = kstrdup(order, GFP_KERNEL);
        next = sep;
        /* Walk the list, looking for matching LSMs. */
        while ((name = strsep(&next, ",")) != NULL) {
                bool found = false;

                for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
                        if (strcmp(lsm->name, name) == 0) {
                                if (lsm->order == LSM_ORDER_MUTABLE)
                                        append_ordered_lsm(lsm, origin);
                                found = true;
                        }
                }

                if (!found)
                        init_debug("%s ignored: %s (not built into kernel)\n",
                                   origin, name);
        }

        /* Process "security=", if given. */
        if (chosen_major_lsm) {
                for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
                        if (exists_ordered_lsm(lsm))
                                continue;
                        if (strcmp(lsm->name, chosen_major_lsm) == 0)
                                append_ordered_lsm(lsm, "security=");
                }
        }

        /* LSM_ORDER_LAST is always last. */
        for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
                if (lsm->order == LSM_ORDER_LAST)
                        append_ordered_lsm(lsm, "   last");
        }

        /* Disable all LSMs not in the ordered list. */
        for (lsm = __start_lsm_info; lsm < __end_lsm_info; lsm++) {
                if (exists_ordered_lsm(lsm))
                        continue;
                set_enabled(lsm, false);
                init_debug("%s skipped: %s (not in requested order)\n",
                           origin, lsm->name);
        }

        kfree(sep);
}

static void __init lsm_static_call_init(struct security_hook_list *hl)
{
        struct lsm_static_call *scall = hl->scalls;
        int i;

        for (i = 0; i < MAX_LSM_COUNT; i++) {
                /* Update the first static call that is not used yet */
                if (!scall->hl) {
                        __static_call_update(scall->key, scall->trampoline,
                                             hl->hook.lsm_func_addr);
                        scall->hl = hl;
                        static_branch_enable(scall->active);
                        return;
                }
                scall++;
        }
        panic("%s - Ran out of static slots.\n", __func__);
}

static void __init lsm_early_cred(struct cred *cred);
static void __init lsm_early_task(struct task_struct *task);

static int lsm_append(const char *new, char **result);

static void __init report_lsm_order(void)
{
        struct lsm_info **lsm, *early;
        int first = 0;

        pr_info("initializing lsm=");

        /* Report each enabled LSM name, comma separated. */
        for (early = __start_early_lsm_info;
             early < __end_early_lsm_info; early++)
                if (is_enabled(early))
                        pr_cont("%s%s", first++ == 0 ? "" : ",", early->name);
        for (lsm = ordered_lsms; *lsm; lsm++)
                if (is_enabled(*lsm))
                        pr_cont("%s%s", first++ == 0 ? "" : ",", (*lsm)->name);

        pr_cont("\n");
}

static void __init ordered_lsm_init(void)
{
        struct lsm_info **lsm;

        if (chosen_lsm_order) {
                if (chosen_major_lsm) {
                        pr_warn("security=%s is ignored because it is superseded by lsm=%s\n",
                                chosen_major_lsm, chosen_lsm_order);
                        chosen_major_lsm = NULL;
                }
                ordered_lsm_parse(chosen_lsm_order, "cmdline");
        } else
                ordered_lsm_parse(builtin_lsm_order, "builtin");

        for (lsm = ordered_lsms; *lsm; lsm++)
                prepare_lsm(*lsm);

        report_lsm_order();

        init_debug("cred blob size       = %d\n", blob_sizes.lbs_cred);
        init_debug("file blob size       = %d\n", blob_sizes.lbs_file);
        init_debug("ib blob size         = %d\n", blob_sizes.lbs_ib);
        init_debug("inode blob size      = %d\n", blob_sizes.lbs_inode);
        init_debug("ipc blob size        = %d\n", blob_sizes.lbs_ipc);
#ifdef CONFIG_KEYS
        init_debug("key blob size        = %d\n", blob_sizes.lbs_key);
#endif /* CONFIG_KEYS */
        init_debug("msg_msg blob size    = %d\n", blob_sizes.lbs_msg_msg);
        init_debug("sock blob size       = %d\n", blob_sizes.lbs_sock);
        init_debug("superblock blob size = %d\n", blob_sizes.lbs_superblock);
        init_debug("perf event blob size = %d\n", blob_sizes.lbs_perf_event);
        init_debug("task blob size       = %d\n", blob_sizes.lbs_task);
        init_debug("tun device blob size = %d\n", blob_sizes.lbs_tun_dev);
        init_debug("xattr slots          = %d\n", blob_sizes.lbs_xattr_count);
        init_debug("bdev blob size       = %d\n", blob_sizes.lbs_bdev);

        /*
         * Create any kmem_caches needed for blobs
         */
        if (blob_sizes.lbs_file)
                lsm_file_cache = kmem_cache_create("lsm_file_cache",
                                                   blob_sizes.lbs_file, 0,
                                                   SLAB_PANIC, NULL);
        if (blob_sizes.lbs_inode)
                lsm_inode_cache = kmem_cache_create("lsm_inode_cache",
                                                    blob_sizes.lbs_inode, 0,
                                                    SLAB_PANIC, NULL);

        lsm_early_cred((struct cred *) current->cred);
        lsm_early_task(current);
        for (lsm = ordered_lsms; *lsm; lsm++)
                initialize_lsm(*lsm);
}

int __init early_security_init(void)
{
        struct lsm_info *lsm;

        for (lsm = __start_early_lsm_info; lsm < __end_early_lsm_info; lsm++) {
                if (!lsm->enabled)
                        lsm->enabled = &lsm_enabled_true;
                prepare_lsm(lsm);
                initialize_lsm(lsm);
        }

        return 0;
}

/**
 * security_init - initializes the security framework
 *
 * This should be called early in the kernel initialization sequence.
 */
int __init security_init(void)
{
        struct lsm_info *lsm;

        init_debug("legacy security=%s\n", chosen_major_lsm ? : " *unspecified*");
        init_debug("  CONFIG_LSM=%s\n", builtin_lsm_order);
        init_debug("boot arg lsm=%s\n", chosen_lsm_order ? : " *unspecified*");

        /*
         * Append the names of the early LSM modules now that kmalloc() is
         * available
         */
        for (lsm = __start_early_lsm_info; lsm < __end_early_lsm_info; lsm++) {
                init_debug("  early started: %s (%s)\n", lsm->name,
                           is_enabled(lsm) ? "enabled" : "disabled");
                if (lsm->enabled)
                        lsm_append(lsm->name, &lsm_names);
        }

        /* Load LSMs in specified order. */
        ordered_lsm_init();

        return 0;
}

/* Save user chosen LSM */
static int __init choose_major_lsm(char *str)
{
        chosen_major_lsm = str;
        return 1;
}
__setup("security=", choose_major_lsm);

/* Explicitly choose LSM initialization order. */
static int __init choose_lsm_order(char *str)
{
        chosen_lsm_order = str;
        return 1;
}
__setup("lsm=", choose_lsm_order);

/* Enable LSM order debugging. */
static int __init enable_debug(char *str)
{
        debug = true;
        return 1;
}
__setup("lsm.debug", enable_debug);

static bool match_last_lsm(const char *list, const char *lsm)
{
        const char *last;

        if (WARN_ON(!list || !lsm))
                return false;
        last = strrchr(list, ',');
        if (last)
                /* Pass the comma, strcmp() will check for '\0' */
                last++;
        else
                last = list;
        return !strcmp(last, lsm);
}

static int lsm_append(const char *new, char **result)
{
        char *cp;

        if (*result == NULL) {
                *result = kstrdup(new, GFP_KERNEL);
                if (*result == NULL)
                        return -ENOMEM;
        } else {
                /* Check if it is the last registered name */
                if (match_last_lsm(*result, new))
                        return 0;
                cp = kasprintf(GFP_KERNEL, "%s,%s", *result, new);
                if (cp == NULL)
                        return -ENOMEM;
                kfree(*result);
                *result = cp;
        }
        return 0;
}

/**
 * security_add_hooks - Add a modules hooks to the hook lists.
 * @hooks: the hooks to add
 * @count: the number of hooks to add
 * @lsmid: the identification information for the security module
 *
 * Each LSM has to register its hooks with the infrastructure.
 */
void __init security_add_hooks(struct security_hook_list *hooks, int count,
                               const struct lsm_id *lsmid)
{
        int i;

        /*
         * A security module may call security_add_hooks() more
         * than once during initialization, and LSM initialization
         * is serialized. Landlock is one such case.
         * Look at the previous entry, if there is one, for duplication.
         */
        if (lsm_active_cnt == 0 || lsm_idlist[lsm_active_cnt - 1] != lsmid) {
                if (lsm_active_cnt >= MAX_LSM_COUNT)
                        panic("%s Too many LSMs registered.\n", __func__);
                lsm_idlist[lsm_active_cnt++] = lsmid;
        }

        for (i = 0; i < count; i++) {
                hooks[i].lsmid = lsmid;
                lsm_static_call_init(&hooks[i]);
        }

        /*
         * Don't try to append during early_security_init(), we'll come back
         * and fix this up afterwards.
         */
        if (slab_is_available()) {
                if (lsm_append(lsmid->name, &lsm_names) < 0)
                        panic("%s - Cannot get early memory.\n", __func__);
        }
}

int call_blocking_lsm_notifier(enum lsm_event event, void *data)
{
        return blocking_notifier_call_chain(&blocking_lsm_notifier_chain,
                                            event, data);
}
EXPORT_SYMBOL(call_blocking_lsm_notifier);

int register_blocking_lsm_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&blocking_lsm_notifier_chain,
                                                nb);
}
EXPORT_SYMBOL(register_blocking_lsm_notifier);

int unregister_blocking_lsm_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&blocking_lsm_notifier_chain,
                                                  nb);
}
EXPORT_SYMBOL(unregister_blocking_lsm_notifier);

/**
 * lsm_blob_alloc - allocate a composite blob
 * @dest: the destination for the blob
 * @size: the size of the blob
 * @gfp: allocation type
 *
 * Allocate a blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_blob_alloc(void **dest, size_t size, gfp_t gfp)
{
        if (size == 0) {
                *dest = NULL;
                return 0;
        }

        *dest = kzalloc(size, gfp);
        if (*dest == NULL)
                return -ENOMEM;
        return 0;
}

/**
 * lsm_cred_alloc - allocate a composite cred blob
 * @cred: the cred that needs a blob
 * @gfp: allocation type
 *
 * Allocate the cred blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_cred_alloc(struct cred *cred, gfp_t gfp)
{
        return lsm_blob_alloc(&cred->security, blob_sizes.lbs_cred, gfp);
}

/**
 * lsm_early_cred - during initialization allocate a composite cred blob
 * @cred: the cred that needs a blob
 *
 * Allocate the cred blob for all the modules
 */
static void __init lsm_early_cred(struct cred *cred)
{
        int rc = lsm_cred_alloc(cred, GFP_KERNEL);

        if (rc)
                panic("%s: Early cred alloc failed.\n", __func__);
}

/**
 * lsm_file_alloc - allocate a composite file blob
 * @file: the file that needs a blob
 *
 * Allocate the file blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_file_alloc(struct file *file)
{
        if (!lsm_file_cache) {
                file->f_security = NULL;
                return 0;
        }

        file->f_security = kmem_cache_zalloc(lsm_file_cache, GFP_KERNEL);
        if (file->f_security == NULL)
                return -ENOMEM;
        return 0;
}

/**
 * lsm_inode_alloc - allocate a composite inode blob
 * @inode: the inode that needs a blob
 * @gfp: allocation flags
 *
 * Allocate the inode blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_inode_alloc(struct inode *inode, gfp_t gfp)
{
        if (!lsm_inode_cache) {
                inode->i_security = NULL;
                return 0;
        }

        inode->i_security = kmem_cache_zalloc(lsm_inode_cache, gfp);
        if (inode->i_security == NULL)
                return -ENOMEM;
        return 0;
}

/**
 * lsm_task_alloc - allocate a composite task blob
 * @task: the task that needs a blob
 *
 * Allocate the task blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_task_alloc(struct task_struct *task)
{
        return lsm_blob_alloc(&task->security, blob_sizes.lbs_task, GFP_KERNEL);
}

/**
 * lsm_ipc_alloc - allocate a composite ipc blob
 * @kip: the ipc that needs a blob
 *
 * Allocate the ipc blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_ipc_alloc(struct kern_ipc_perm *kip)
{
        return lsm_blob_alloc(&kip->security, blob_sizes.lbs_ipc, GFP_KERNEL);
}

#ifdef CONFIG_KEYS
/**
 * lsm_key_alloc - allocate a composite key blob
 * @key: the key that needs a blob
 *
 * Allocate the key blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_key_alloc(struct key *key)
{
        return lsm_blob_alloc(&key->security, blob_sizes.lbs_key, GFP_KERNEL);
}
#endif /* CONFIG_KEYS */

/**
 * lsm_msg_msg_alloc - allocate a composite msg_msg blob
 * @mp: the msg_msg that needs a blob
 *
 * Allocate the ipc blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_msg_msg_alloc(struct msg_msg *mp)
{
        return lsm_blob_alloc(&mp->security, blob_sizes.lbs_msg_msg,
                              GFP_KERNEL);
}

/**
 * lsm_bdev_alloc - allocate a composite block_device blob
 * @bdev: the block_device that needs a blob
 *
 * Allocate the block_device blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_bdev_alloc(struct block_device *bdev)
{
        if (blob_sizes.lbs_bdev == 0) {
                bdev->bd_security = NULL;
                return 0;
        }

        bdev->bd_security = kzalloc(blob_sizes.lbs_bdev, GFP_KERNEL);
        if (!bdev->bd_security)
                return -ENOMEM;

        return 0;
}

/**
 * lsm_early_task - during initialization allocate a composite task blob
 * @task: the task that needs a blob
 *
 * Allocate the task blob for all the modules
 */
static void __init lsm_early_task(struct task_struct *task)
{
        int rc = lsm_task_alloc(task);

        if (rc)
                panic("%s: Early task alloc failed.\n", __func__);
}

/**
 * lsm_superblock_alloc - allocate a composite superblock blob
 * @sb: the superblock that needs a blob
 *
 * Allocate the superblock blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_superblock_alloc(struct super_block *sb)
{
        return lsm_blob_alloc(&sb->s_security, blob_sizes.lbs_superblock,
                              GFP_KERNEL);
}

/**
 * lsm_fill_user_ctx - Fill a user space lsm_ctx structure
 * @uctx: a userspace LSM context to be filled
 * @uctx_len: available uctx size (input), used uctx size (output)
 * @val: the new LSM context value
 * @val_len: the size of the new LSM context value
 * @id: LSM id
 * @flags: LSM defined flags
 *
 * Fill all of the fields in a userspace lsm_ctx structure.  If @uctx is NULL
 * simply calculate the required size to output via @utc_len and return
 * success.
 *
 * Returns 0 on success, -E2BIG if userspace buffer is not large enough,
 * -EFAULT on a copyout error, -ENOMEM if memory can't be allocated.
 */
int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, u32 *uctx_len,
                      void *val, size_t val_len,
                      u64 id, u64 flags)
{
        struct lsm_ctx *nctx = NULL;
        size_t nctx_len;
        int rc = 0;

        nctx_len = ALIGN(struct_size(nctx, ctx, val_len), sizeof(void *));
        if (nctx_len > *uctx_len) {
                rc = -E2BIG;
                goto out;
        }

        /* no buffer - return success/0 and set @uctx_len to the req size */
        if (!uctx)
                goto out;

        nctx = kzalloc(nctx_len, GFP_KERNEL);
        if (nctx == NULL) {
                rc = -ENOMEM;
                goto out;
        }
        nctx->id = id;
        nctx->flags = flags;
        nctx->len = nctx_len;
        nctx->ctx_len = val_len;
        memcpy(nctx->ctx, val, val_len);

        if (copy_to_user(uctx, nctx, nctx_len))
                rc = -EFAULT;

out:
        kfree(nctx);
        *uctx_len = nctx_len;
        return rc;
}

/*
 * The default value of the LSM hook is defined in linux/lsm_hook_defs.h and
 * can be accessed with:
 *
 *        LSM_RET_DEFAULT(<hook_name>)
 *
 * The macros below define static constants for the default value of each
 * LSM hook.
 */
#define LSM_RET_DEFAULT(NAME) (NAME##_default)
#define DECLARE_LSM_RET_DEFAULT_void(DEFAULT, NAME)
#define DECLARE_LSM_RET_DEFAULT_int(DEFAULT, NAME) \
        static const int __maybe_unused LSM_RET_DEFAULT(NAME) = (DEFAULT);
#define LSM_HOOK(RET, DEFAULT, NAME, ...) \
        DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)

#include <linux/lsm_hook_defs.h>
#undef LSM_HOOK

/*
 * Hook list operation macros.
 *
 * call_void_hook:
 *        This is a hook that does not return a value.
 *
 * call_int_hook:
 *        This is a hook that returns a value.
 */
#define __CALL_STATIC_VOID(NUM, HOOK, ...)                                     \
do {                                                                             \
        if (static_branch_unlikely(&SECURITY_HOOK_ACTIVE_KEY(HOOK, NUM))) {    \
                static_call(LSM_STATIC_CALL(HOOK, NUM))(__VA_ARGS__);             \
        }                                                                     \
} while (0);

#define call_void_hook(HOOK, ...)                                 \
        do {                                                      \
                LSM_LOOP_UNROLL(__CALL_STATIC_VOID, HOOK, __VA_ARGS__); \
        } while (0)


#define __CALL_STATIC_INT(NUM, R, HOOK, LABEL, ...)                             \
do {                                                                             \
        if (static_branch_unlikely(&SECURITY_HOOK_ACTIVE_KEY(HOOK, NUM))) {  \
                R = static_call(LSM_STATIC_CALL(HOOK, NUM))(__VA_ARGS__);    \
                if (R != LSM_RET_DEFAULT(HOOK))                                     \
                        goto LABEL;                                             \
        }                                                                     \
} while (0);

#define call_int_hook(HOOK, ...)                                        \
({                                                                        \
        __label__ OUT;                                                        \
        int RC = LSM_RET_DEFAULT(HOOK);                                        \
                                                                        \
        LSM_LOOP_UNROLL(__CALL_STATIC_INT, RC, HOOK, OUT, __VA_ARGS__);        \
OUT:                                                                        \
        RC;                                                                \
})

#define lsm_for_each_hook(scall, NAME)                                        \
        for (scall = static_calls_table.NAME;                                \
             scall - static_calls_table.NAME < MAX_LSM_COUNT; scall++)  \
                if (static_key_enabled(&scall->active->key))

/* Security operations */

/**
 * security_binder_set_context_mgr() - Check if becoming binder ctx mgr is ok
 * @mgr: task credentials of current binder process
 *
 * Check whether @mgr is allowed to be the binder context manager.
 *
 * Return: Return 0 if permission is granted.
 */
int security_binder_set_context_mgr(const struct cred *mgr)
{
        return call_int_hook(binder_set_context_mgr, mgr);
}

/**
 * security_binder_transaction() - Check if a binder transaction is allowed
 * @from: sending process
 * @to: receiving process
 *
 * Check whether @from is allowed to invoke a binder transaction call to @to.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_binder_transaction(const struct cred *from,
                                const struct cred *to)
{
        return call_int_hook(binder_transaction, from, to);
}

/**
 * security_binder_transfer_binder() - Check if a binder transfer is allowed
 * @from: sending process
 * @to: receiving process
 *
 * Check whether @from is allowed to transfer a binder reference to @to.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_binder_transfer_binder(const struct cred *from,
                                    const struct cred *to)
{
        return call_int_hook(binder_transfer_binder, from, to);
}

/**
 * security_binder_transfer_file() - Check if a binder file xfer is allowed
 * @from: sending process
 * @to: receiving process
 * @file: file being transferred
 *
 * Check whether @from is allowed to transfer @file to @to.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_binder_transfer_file(const struct cred *from,
                                  const struct cred *to, const struct file *file)
{
        return call_int_hook(binder_transfer_file, from, to, file);
}

/**
 * security_ptrace_access_check() - Check if tracing is allowed
 * @child: target process
 * @mode: PTRACE_MODE flags
 *
 * Check permission before allowing the current process to trace the @child
 * process.  Security modules may also want to perform a process tracing check
 * during an execve in the set_security or apply_creds hooks of tracing check
 * during an execve in the bprm_set_creds hook of binprm_security_ops if the
 * process is being traced and its security attributes would be changed by the
 * execve.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_ptrace_access_check(struct task_struct *child, unsigned int mode)
{
        return call_int_hook(ptrace_access_check, child, mode);
}

/**
 * security_ptrace_traceme() - Check if tracing is allowed
 * @parent: tracing process
 *
 * Check that the @parent process has sufficient permission to trace the
 * current process before allowing the current process to present itself to the
 * @parent process for tracing.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_ptrace_traceme(struct task_struct *parent)
{
        return call_int_hook(ptrace_traceme, parent);
}

/**
 * security_capget() - Get the capability sets for a process
 * @target: target process
 * @effective: effective capability set
 * @inheritable: inheritable capability set
 * @permitted: permitted capability set
 *
 * Get the @effective, @inheritable, and @permitted capability sets for the
 * @target process.  The hook may also perform permission checking to determine
 * if the current process is allowed to see the capability sets of the @target
 * process.
 *
 * Return: Returns 0 if the capability sets were successfully obtained.
 */
int security_capget(const struct task_struct *target,
                    kernel_cap_t *effective,
                    kernel_cap_t *inheritable,
                    kernel_cap_t *permitted)
{
        return call_int_hook(capget, target, effective, inheritable, permitted);
}

/**
 * security_capset() - Set the capability sets for a process
 * @new: new credentials for the target process
 * @old: current credentials of the target process
 * @effective: effective capability set
 * @inheritable: inheritable capability set
 * @permitted: permitted capability set
 *
 * Set the @effective, @inheritable, and @permitted capability sets for the
 * current process.
 *
 * Return: Returns 0 and update @new if permission is granted.
 */
int security_capset(struct cred *new, const struct cred *old,
                    const kernel_cap_t *effective,
                    const kernel_cap_t *inheritable,
                    const kernel_cap_t *permitted)
{
        return call_int_hook(capset, new, old, effective, inheritable,
                             permitted);
}

/**
 * security_capable() - Check if a process has the necessary capability
 * @cred: credentials to examine
 * @ns: user namespace
 * @cap: capability requested
 * @opts: capability check options
 *
 * Check whether the @tsk process has the @cap capability in the indicated
 * credentials.  @cap contains the capability <include/linux/capability.h>.
 * @opts contains options for the capable check <include/linux/security.h>.
 *
 * Return: Returns 0 if the capability is granted.
 */
int security_capable(const struct cred *cred,
                     struct user_namespace *ns,
                     int cap,
                     unsigned int opts)
{
        return call_int_hook(capable, cred, ns, cap, opts);
}

/**
 * security_quotactl() - Check if a quotactl() syscall is allowed for this fs
 * @cmds: commands
 * @type: type
 * @id: id
 * @sb: filesystem
 *
 * Check whether the quotactl syscall is allowed for this @sb.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_quotactl(int cmds, int type, int id, const struct super_block *sb)
{
        return call_int_hook(quotactl, cmds, type, id, sb);
}

/**
 * security_quota_on() - Check if QUOTAON is allowed for a dentry
 * @dentry: dentry
 *
 * Check whether QUOTAON is allowed for @dentry.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_quota_on(struct dentry *dentry)
{
        return call_int_hook(quota_on, dentry);
}

/**
 * security_syslog() - Check if accessing the kernel message ring is allowed
 * @type: SYSLOG_ACTION_* type
 *
 * Check permission before accessing the kernel message ring or changing
 * logging to the console.  See the syslog(2) manual page for an explanation of
 * the @type values.
 *
 * Return: Return 0 if permission is granted.
 */
int security_syslog(int type)
{
        return call_int_hook(syslog, type);
}

/**
 * security_settime64() - Check if changing the system time is allowed
 * @ts: new time
 * @tz: timezone
 *
 * Check permission to change the system time, struct timespec64 is defined in
 * <include/linux/time64.h> and timezone is defined in <include/linux/time.h>.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_settime64(const struct timespec64 *ts, const struct timezone *tz)
{
        return call_int_hook(settime, ts, tz);
}

/**
 * security_vm_enough_memory_mm() - Check if allocating a new mem map is allowed
 * @mm: mm struct
 * @pages: number of pages
 *
 * Check permissions for allocating a new virtual mapping.  If all LSMs return
 * a positive value, __vm_enough_memory() will be called with cap_sys_admin
 * set. If at least one LSM returns 0 or negative, __vm_enough_memory() will be
 * called with cap_sys_admin cleared.
 *
 * Return: Returns 0 if permission is granted by the LSM infrastructure to the
 *         caller.
 */
int security_vm_enough_memory_mm(struct mm_struct *mm, long pages)
{
        struct lsm_static_call *scall;
        int cap_sys_admin = 1;
        int rc;

        /*
         * The module will respond with 0 if it thinks the __vm_enough_memory()
         * call should be made with the cap_sys_admin set. If all of the modules
         * agree that it should be set it will. If any module thinks it should
         * not be set it won't.
         */
        lsm_for_each_hook(scall, vm_enough_memory) {
                rc = scall->hl->hook.vm_enough_memory(mm, pages);
                if (rc < 0) {
                        cap_sys_admin = 0;
                        break;
                }
        }
        return __vm_enough_memory(mm, pages, cap_sys_admin);
}

/**
 * security_bprm_creds_for_exec() - Prepare the credentials for exec()
 * @bprm: binary program information
 *
 * If the setup in prepare_exec_creds did not setup @bprm->cred->security
 * properly for executing @bprm->file, update the LSM's portion of
 * @bprm->cred->security to be what commit_creds needs to install for the new
 * program.  This hook may also optionally check permissions (e.g. for
 * transitions between security domains).  The hook must set @bprm->secureexec
 * to 1 if AT_SECURE should be set to request libc enable secure mode.  @bprm
 * contains the linux_binprm structure.
 *
 * Return: Returns 0 if the hook is successful and permission is granted.
 */
int security_bprm_creds_for_exec(struct linux_binprm *bprm)
{
        return call_int_hook(bprm_creds_for_exec, bprm);
}

/**
 * security_bprm_creds_from_file() - Update linux_binprm creds based on file
 * @bprm: binary program information
 * @file: associated file
 *
 * If @file is setpcap, suid, sgid or otherwise marked to change privilege upon
 * exec, update @bprm->cred to reflect that change. This is called after
 * finding the binary that will be executed without an interpreter.  This
 * ensures that the credentials will not be derived from a script that the
 * binary will need to reopen, which when reopend may end up being a completely
 * different file.  This hook may also optionally check permissions (e.g. for
 * transitions between security domains).  The hook must set @bprm->secureexec
 * to 1 if AT_SECURE should be set to request libc enable secure mode.  The
 * hook must add to @bprm->per_clear any personality flags that should be
 * cleared from current->personality.  @bprm contains the linux_binprm
 * structure.
 *
 * Return: Returns 0 if the hook is successful and permission is granted.
 */
int security_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file)
{
        return call_int_hook(bprm_creds_from_file, bprm, file);
}

/**
 * security_bprm_check() - Mediate binary handler search
 * @bprm: binary program information
 *
 * This hook mediates the point when a search for a binary handler will begin.
 * It allows a check against the @bprm->cred->security value which was set in
 * the preceding creds_for_exec call.  The argv list and envp list are reliably
 * available in @bprm.  This hook may be called multiple times during a single
 * execve.  @bprm contains the linux_binprm structure.
 *
 * Return: Returns 0 if the hook is successful and permission is granted.
 */
int security_bprm_check(struct linux_binprm *bprm)
{
        return call_int_hook(bprm_check_security, bprm);
}

/**
 * security_bprm_committing_creds() - Install creds for a process during exec()
 * @bprm: binary program information
 *
 * Prepare to install the new security attributes of a process being
 * transformed by an execve operation, based on the old credentials pointed to
 * by @current->cred and the information set in @bprm->cred by the
 * bprm_creds_for_exec hook.  @bprm points to the linux_binprm structure.  This
 * hook is a good place to perform state changes on the process such as closing
 * open file descriptors to which access will no longer be granted when the
 * attributes are changed.  This is called immediately before commit_creds().
 */
void security_bprm_committing_creds(const struct linux_binprm *bprm)
{
        call_void_hook(bprm_committing_creds, bprm);
}

/**
 * security_bprm_committed_creds() - Tidy up after cred install during exec()
 * @bprm: binary program information
 *
 * Tidy up after the installation of the new security attributes of a process
 * being transformed by an execve operation.  The new credentials have, by this
 * point, been set to @current->cred.  @bprm points to the linux_binprm
 * structure.  This hook is a good place to perform state changes on the
 * process such as clearing out non-inheritable signal state.  This is called
 * immediately after commit_creds().
 */
void security_bprm_committed_creds(const struct linux_binprm *bprm)
{
        call_void_hook(bprm_committed_creds, bprm);
}

/**
 * security_fs_context_submount() - Initialise fc->security
 * @fc: new filesystem context
 * @reference: dentry reference for submount/remount
 *
 * Fill out the ->security field for a new fs_context.
 *
 * Return: Returns 0 on success or negative error code on failure.
 */
int security_fs_context_submount(struct fs_context *fc, struct super_block *reference)
{
        return call_int_hook(fs_context_submount, fc, reference);
}

/**
 * security_fs_context_dup() - Duplicate a fs_context LSM blob
 * @fc: destination filesystem context
 * @src_fc: source filesystem context
 *
 * Allocate and attach a security structure to sc->security.  This pointer is
 * initialised to NULL by the caller.  @fc indicates the new filesystem context.
 * @src_fc indicates the original filesystem context.
 *
 * Return: Returns 0 on success or a negative error code on failure.
 */
int security_fs_context_dup(struct fs_context *fc, struct fs_context *src_fc)
{
        return call_int_hook(fs_context_dup, fc, src_fc);
}

/**
 * security_fs_context_parse_param() - Configure a filesystem context
 * @fc: filesystem context
 * @param: filesystem parameter
 *
 * Userspace provided a parameter to configure a superblock.  The LSM can
 * consume the parameter or return it to the caller for use elsewhere.
 *
 * Return: If the parameter is used by the LSM it should return 0, if it is
 *         returned to the caller -ENOPARAM is returned, otherwise a negative
 *         error code is returned.
 */
int security_fs_context_parse_param(struct fs_context *fc,
                                    struct fs_parameter *param)
{
        struct lsm_static_call *scall;
        int trc;
        int rc = -ENOPARAM;

        lsm_for_each_hook(scall, fs_context_parse_param) {
                trc = scall->hl->hook.fs_context_parse_param(fc, param);
                if (trc == 0)
                        rc = 0;
                else if (trc != -ENOPARAM)
                        return trc;
        }
        return rc;
}

/**
 * security_sb_alloc() - Allocate a super_block LSM blob
 * @sb: filesystem superblock
 *
 * Allocate and attach a security structure to the sb->s_security field.  The
 * s_security field is initialized to NULL when the structure is allocated.
 * @sb contains the super_block structure to be modified.
 *
 * Return: Returns 0 if operation was successful.
 */
int security_sb_alloc(struct super_block *sb)
{
        int rc = lsm_superblock_alloc(sb);

        if (unlikely(rc))
                return rc;
        rc = call_int_hook(sb_alloc_security, sb);
        if (unlikely(rc))
                security_sb_free(sb);
        return rc;
}

/**
 * security_sb_delete() - Release super_block LSM associated objects
 * @sb: filesystem superblock
 *
 * Release objects tied to a superblock (e.g. inodes).  @sb contains the
 * super_block structure being released.
 */
void security_sb_delete(struct super_block *sb)
{
        call_void_hook(sb_delete, sb);
}

/**
 * security_sb_free() - Free a super_block LSM blob
 * @sb: filesystem superblock
 *
 * Deallocate and clear the sb->s_security field.  @sb contains the super_block
 * structure to be modified.
 */
void security_sb_free(struct super_block *sb)
{
        call_void_hook(sb_free_security, sb);
        kfree(sb->s_security);
        sb->s_security = NULL;
}

/**
 * security_free_mnt_opts() - Free memory associated with mount options
 * @mnt_opts: LSM processed mount options
 *
 * Free memory associated with @mnt_ops.
 */
void security_free_mnt_opts(void **mnt_opts)
{
        if (!*mnt_opts)
                return;
        call_void_hook(sb_free_mnt_opts, *mnt_opts);
        *mnt_opts = NULL;
}
EXPORT_SYMBOL(security_free_mnt_opts);

/**
 * security_sb_eat_lsm_opts() - Consume LSM mount options
 * @options: mount options
 * @mnt_opts: LSM processed mount options
 *
 * Eat (scan @options) and save them in @mnt_opts.
 *
 * Return: Returns 0 on success, negative values on failure.
 */
int security_sb_eat_lsm_opts(char *options, void **mnt_opts)
{
        return call_int_hook(sb_eat_lsm_opts, options, mnt_opts);
}
EXPORT_SYMBOL(security_sb_eat_lsm_opts);

/**
 * security_sb_mnt_opts_compat() - Check if new mount options are allowed
 * @sb: filesystem superblock
 * @mnt_opts: new mount options
 *
 * Determine if the new mount options in @mnt_opts are allowed given the
 * existing mounted filesystem at @sb.  @sb superblock being compared.
 *
 * Return: Returns 0 if options are compatible.
 */
int security_sb_mnt_opts_compat(struct super_block *sb,
                                void *mnt_opts)
{
        return call_int_hook(sb_mnt_opts_compat, sb, mnt_opts);
}
EXPORT_SYMBOL(security_sb_mnt_opts_compat);

/**
 * security_sb_remount() - Verify no incompatible mount changes during remount
 * @sb: filesystem superblock
 * @mnt_opts: (re)mount options
 *
 * Extracts security system specific mount options and verifies no changes are
 * being made to those options.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sb_remount(struct super_block *sb,
                        void *mnt_opts)
{
        return call_int_hook(sb_remount, sb, mnt_opts);
}
EXPORT_SYMBOL(security_sb_remount);

/**
 * security_sb_kern_mount() - Check if a kernel mount is allowed
 * @sb: filesystem superblock
 *
 * Mount this @sb if allowed by permissions.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sb_kern_mount(const struct super_block *sb)
{
        return call_int_hook(sb_kern_mount, sb);
}

/**
 * security_sb_show_options() - Output the mount options for a superblock
 * @m: output file
 * @sb: filesystem superblock
 *
 * Show (print on @m) mount options for this @sb.
 *
 * Return: Returns 0 on success, negative values on failure.
 */
int security_sb_show_options(struct seq_file *m, struct super_block *sb)
{
        return call_int_hook(sb_show_options, m, sb);
}

/**
 * security_sb_statfs() - Check if accessing fs stats is allowed
 * @dentry: superblock handle
 *
 * Check permission before obtaining filesystem statistics for the @mnt
 * mountpoint.  @dentry is a handle on the superblock for the filesystem.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sb_statfs(struct dentry *dentry)
{
        return call_int_hook(sb_statfs, dentry);
}

/**
 * security_sb_mount() - Check permission for mounting a filesystem
 * @dev_name: filesystem backing device
 * @path: mount point
 * @type: filesystem type
 * @flags: mount flags
 * @data: filesystem specific data
 *
 * Check permission before an object specified by @dev_name is mounted on the
 * mount point named by @nd.  For an ordinary mount, @dev_name identifies a
 * device if the file system type requires a device.  For a remount
 * (@flags & MS_REMOUNT), @dev_name is irrelevant.  For a loopback/bind mount
 * (@flags & MS_BIND), @dev_name identifies the        pathname of the object being
 * mounted.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sb_mount(const char *dev_name, const struct path *path,
                      const char *type, unsigned long flags, void *data)
{
        return call_int_hook(sb_mount, dev_name, path, type, flags, data);
}

/**
 * security_sb_umount() - Check permission for unmounting a filesystem
 * @mnt: mounted filesystem
 * @flags: unmount flags
 *
 * Check permission before the @mnt file system is unmounted.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sb_umount(struct vfsmount *mnt, int flags)
{
        return call_int_hook(sb_umount, mnt, flags);
}

/**
 * security_sb_pivotroot() - Check permissions for pivoting the rootfs
 * @old_path: new location for current rootfs
 * @new_path: location of the new rootfs
 *
 * Check permission before pivoting the root filesystem.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sb_pivotroot(const struct path *old_path,
                          const struct path *new_path)
{
        return call_int_hook(sb_pivotroot, old_path, new_path);
}

/**
 * security_sb_set_mnt_opts() - Set the mount options for a filesystem
 * @sb: filesystem superblock
 * @mnt_opts: binary mount options
 * @kern_flags: kernel flags (in)
 * @set_kern_flags: kernel flags (out)
 *
 * Set the security relevant mount options used for a superblock.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_sb_set_mnt_opts(struct super_block *sb,
                             void *mnt_opts,
                             unsigned long kern_flags,
                             unsigned long *set_kern_flags)
{
        struct lsm_static_call *scall;
        int rc = mnt_opts ? -EOPNOTSUPP : LSM_RET_DEFAULT(sb_set_mnt_opts);

        lsm_for_each_hook(scall, sb_set_mnt_opts) {
                rc = scall->hl->hook.sb_set_mnt_opts(sb, mnt_opts, kern_flags,
                                              set_kern_flags);
                if (rc != LSM_RET_DEFAULT(sb_set_mnt_opts))
                        break;
        }
        return rc;
}
EXPORT_SYMBOL(security_sb_set_mnt_opts);

/**
 * security_sb_clone_mnt_opts() - Duplicate superblock mount options
 * @oldsb: source superblock
 * @newsb: destination superblock
 * @kern_flags: kernel flags (in)
 * @set_kern_flags: kernel flags (out)
 *
 * Copy all security options from a given superblock to another.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_sb_clone_mnt_opts(const struct super_block *oldsb,
                               struct super_block *newsb,
                               unsigned long kern_flags,
                               unsigned long *set_kern_flags)
{
        return call_int_hook(sb_clone_mnt_opts, oldsb, newsb,
                             kern_flags, set_kern_flags);
}
EXPORT_SYMBOL(security_sb_clone_mnt_opts);

/**
 * security_move_mount() - Check permissions for moving a mount
 * @from_path: source mount point
 * @to_path: destination mount point
 *
 * Check permission before a mount is moved.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_move_mount(const struct path *from_path,
                        const struct path *to_path)
{
        return call_int_hook(move_mount, from_path, to_path);
}

/**
 * security_path_notify() - Check if setting a watch is allowed
 * @path: file path
 * @mask: event mask
 * @obj_type: file path type
 *
 * Check permissions before setting a watch on events as defined by @mask, on
 * an object at @path, whose type is defined by @obj_type.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_notify(const struct path *path, u64 mask,
                         unsigned int obj_type)
{
        return call_int_hook(path_notify, path, mask, obj_type);
}

/**
 * security_inode_alloc() - Allocate an inode LSM blob
 * @inode: the inode
 * @gfp: allocation flags
 *
 * Allocate and attach a security structure to @inode->i_security.  The
 * i_security field is initialized to NULL when the inode structure is
 * allocated.
 *
 * Return: Return 0 if operation was successful.
 */
int security_inode_alloc(struct inode *inode, gfp_t gfp)
{
        int rc = lsm_inode_alloc(inode, gfp);

        if (unlikely(rc))
                return rc;
        rc = call_int_hook(inode_alloc_security, inode);
        if (unlikely(rc))
                security_inode_free(inode);
        return rc;
}

static void inode_free_by_rcu(struct rcu_head *head)
{
        /* The rcu head is at the start of the inode blob */
        call_void_hook(inode_free_security_rcu, head);
        kmem_cache_free(lsm_inode_cache, head);
}

/**
 * security_inode_free() - Free an inode's LSM blob
 * @inode: the inode
 *
 * Release any LSM resources associated with @inode, although due to the
 * inode's RCU protections it is possible that the resources will not be
 * fully released until after the current RCU grace period has elapsed.
 *
 * It is important for LSMs to note that despite being present in a call to
 * security_inode_free(), @inode may still be referenced in a VFS path walk
 * and calls to security_inode_permission() may be made during, or after,
 * a call to security_inode_free().  For this reason the inode->i_security
 * field is released via a call_rcu() callback and any LSMs which need to
 * retain inode state for use in security_inode_permission() should only
 * release that state in the inode_free_security_rcu() LSM hook callback.
 */
void security_inode_free(struct inode *inode)
{
        call_void_hook(inode_free_security, inode);
        if (!inode->i_security)
                return;
        call_rcu((struct rcu_head *)inode->i_security, inode_free_by_rcu);
}

/**
 * security_dentry_init_security() - Perform dentry initialization
 * @dentry: the dentry to initialize
 * @mode: mode used to determine resource type
 * @name: name of the last path component
 * @xattr_name: name of the security/LSM xattr
 * @ctx: pointer to the resulting LSM context
 * @ctxlen: length of @ctx
 *
 * Compute a context for a dentry as the inode is not yet available since NFSv4
 * has no label backed by an EA anyway.  It is important to note that
 * @xattr_name does not need to be free'd by the caller, it is a static string.
 *
 * Return: Returns 0 on success, negative values on failure.
 */
int security_dentry_init_security(struct dentry *dentry, int mode,
                                  const struct qstr *name,
                                  const char **xattr_name, void **ctx,
                                  u32 *ctxlen)
{
        return call_int_hook(dentry_init_security, dentry, mode, name,
                             xattr_name, ctx, ctxlen);
}
EXPORT_SYMBOL(security_dentry_init_security);

/**
 * security_dentry_create_files_as() - Perform dentry initialization
 * @dentry: the dentry to initialize
 * @mode: mode used to determine resource type
 * @name: name of the last path component
 * @old: creds to use for LSM context calculations
 * @new: creds to modify
 *
 * Compute a context for a dentry as the inode is not yet available and set
 * that context in passed in creds so that new files are created using that
 * context. Context is calculated using the passed in creds and not the creds
 * of the caller.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_dentry_create_files_as(struct dentry *dentry, int mode,
                                    struct qstr *name,
                                    const struct cred *old, struct cred *new)
{
        return call_int_hook(dentry_create_files_as, dentry, mode,
                             name, old, new);
}
EXPORT_SYMBOL(security_dentry_create_files_as);

/**
 * security_inode_init_security() - Initialize an inode's LSM context
 * @inode: the inode
 * @dir: parent directory
 * @qstr: last component of the pathname
 * @initxattrs: callback function to write xattrs
 * @fs_data: filesystem specific data
 *
 * Obtain the security attribute name suffix and value to set on a newly
 * created inode and set up the incore security field for the new inode.  This
 * hook is called by the fs code as part of the inode creation transaction and
 * provides for atomic labeling of the inode, unlike the post_create/mkdir/...
 * hooks called by the VFS.
 *
 * The hook function is expected to populate the xattrs array, by calling
 * lsm_get_xattr_slot() to retrieve the slots reserved by the security module
 * with the lbs_xattr_count field of the lsm_blob_sizes structure.  For each
 * slot, the hook function should set ->name to the attribute name suffix
 * (e.g. selinux), to allocate ->value (will be freed by the caller) and set it
 * to the attribute value, to set ->value_len to the length of the value.  If
 * the security module does not use security attributes or does not wish to put
 * a security attribute on this particular inode, then it should return
 * -EOPNOTSUPP to skip this processing.
 *
 * Return: Returns 0 if the LSM successfully initialized all of the inode
 *         security attributes that are required, negative values otherwise.
 */
int security_inode_init_security(struct inode *inode, struct inode *dir,
                                 const struct qstr *qstr,
                                 const initxattrs initxattrs, void *fs_data)
{
        struct lsm_static_call *scall;
        struct xattr *new_xattrs = NULL;
        int ret = -EOPNOTSUPP, xattr_count = 0;

        if (unlikely(IS_PRIVATE(inode)))
                return 0;

        if (!blob_sizes.lbs_xattr_count)
                return 0;

        if (initxattrs) {
                /* Allocate +1 as terminator. */
                new_xattrs = kcalloc(blob_sizes.lbs_xattr_count + 1,
                                     sizeof(*new_xattrs), GFP_NOFS);
                if (!new_xattrs)
                        return -ENOMEM;
        }

        lsm_for_each_hook(scall, inode_init_security) {
                ret = scall->hl->hook.inode_init_security(inode, dir, qstr, new_xattrs,
                                                  &xattr_count);
                if (ret && ret != -EOPNOTSUPP)
                        goto out;
                /*
                 * As documented in lsm_hooks.h, -EOPNOTSUPP in this context
                 * means that the LSM is not willing to provide an xattr, not
                 * that it wants to signal an error. Thus, continue to invoke
                 * the remaining LSMs.
                 */
        }

        /* If initxattrs() is NULL, xattr_count is zero, skip the call. */
        if (!xattr_count)
                goto out;

        ret = initxattrs(inode, new_xattrs, fs_data);
out:
        for (; xattr_count > 0; xattr_count--)
                kfree(new_xattrs[xattr_count - 1].value);
        kfree(new_xattrs);
        return (ret == -EOPNOTSUPP) ? 0 : ret;
}
EXPORT_SYMBOL(security_inode_init_security);

/**
 * security_inode_init_security_anon() - Initialize an anonymous inode
 * @inode: the inode
 * @name: the anonymous inode class
 * @context_inode: an optional related inode
 *
 * Set up the incore security field for the new anonymous inode and return
 * whether the inode creation is permitted by the security module or not.
 *
 * Return: Returns 0 on success, -EACCES if the security module denies the
 * creation of this inode, or another -errno upon other errors.
 */
int security_inode_init_security_anon(struct inode *inode,
                                      const struct qstr *name,
                                      const struct inode *context_inode)
{
        return call_int_hook(inode_init_security_anon, inode, name,
                             context_inode);
}

#ifdef CONFIG_SECURITY_PATH
/**
 * security_path_mknod() - Check if creating a special file is allowed
 * @dir: parent directory
 * @dentry: new file
 * @mode: new file mode
 * @dev: device number
 *
 * Check permissions when creating a file. Note that this hook is called even
 * if mknod operation is being done for a regular file.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_mknod(const struct path *dir, struct dentry *dentry,
                        umode_t mode, unsigned int dev)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
                return 0;
        return call_int_hook(path_mknod, dir, dentry, mode, dev);
}
EXPORT_SYMBOL(security_path_mknod);

/**
 * security_path_post_mknod() - Update inode security after reg file creation
 * @idmap: idmap of the mount
 * @dentry: new file
 *
 * Update inode security field after a regular file has been created.
 */
void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return;
        call_void_hook(path_post_mknod, idmap, dentry);
}

/**
 * security_path_mkdir() - Check if creating a new directory is allowed
 * @dir: parent directory
 * @dentry: new directory
 * @mode: new directory mode
 *
 * Check permissions to create a new directory in the existing directory.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_mkdir(const struct path *dir, struct dentry *dentry,
                        umode_t mode)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
                return 0;
        return call_int_hook(path_mkdir, dir, dentry, mode);
}
EXPORT_SYMBOL(security_path_mkdir);

/**
 * security_path_rmdir() - Check if removing a directory is allowed
 * @dir: parent directory
 * @dentry: directory to remove
 *
 * Check the permission to remove a directory.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_rmdir(const struct path *dir, struct dentry *dentry)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
                return 0;
        return call_int_hook(path_rmdir, dir, dentry);
}

/**
 * security_path_unlink() - Check if removing a hard link is allowed
 * @dir: parent directory
 * @dentry: file
 *
 * Check the permission to remove a hard link to a file.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_unlink(const struct path *dir, struct dentry *dentry)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
                return 0;
        return call_int_hook(path_unlink, dir, dentry);
}
EXPORT_SYMBOL(security_path_unlink);

/**
 * security_path_symlink() - Check if creating a symbolic link is allowed
 * @dir: parent directory
 * @dentry: symbolic link
 * @old_name: file pathname
 *
 * Check the permission to create a symbolic link to a file.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_symlink(const struct path *dir, struct dentry *dentry,
                          const char *old_name)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dir->dentry))))
                return 0;
        return call_int_hook(path_symlink, dir, dentry, old_name);
}

/**
 * security_path_link - Check if creating a hard link is allowed
 * @old_dentry: existing file
 * @new_dir: new parent directory
 * @new_dentry: new link
 *
 * Check permission before creating a new hard link to a file.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_link(struct dentry *old_dentry, const struct path *new_dir,
                       struct dentry *new_dentry)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry))))
                return 0;
        return call_int_hook(path_link, old_dentry, new_dir, new_dentry);
}

/**
 * security_path_rename() - Check if renaming a file is allowed
 * @old_dir: parent directory of the old file
 * @old_dentry: the old file
 * @new_dir: parent directory of the new file
 * @new_dentry: the new file
 * @flags: flags
 *
 * Check for permission to rename a file or directory.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_rename(const struct path *old_dir, struct dentry *old_dentry,
                         const struct path *new_dir, struct dentry *new_dentry,
                         unsigned int flags)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry)) ||
                     (d_is_positive(new_dentry) &&
                      IS_PRIVATE(d_backing_inode(new_dentry)))))
                return 0;

        return call_int_hook(path_rename, old_dir, old_dentry, new_dir,
                             new_dentry, flags);
}
EXPORT_SYMBOL(security_path_rename);

/**
 * security_path_truncate() - Check if truncating a file is allowed
 * @path: file
 *
 * Check permission before truncating the file indicated by path.  Note that
 * truncation permissions may also be checked based on already opened files,
 * using the security_file_truncate() hook.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_truncate(const struct path *path)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry))))
                return 0;
        return call_int_hook(path_truncate, path);
}

/**
 * security_path_chmod() - Check if changing the file's mode is allowed
 * @path: file
 * @mode: new mode
 *
 * Check for permission to change a mode of the file @path. The new mode is
 * specified in @mode which is a bitmask of constants from
 * <include/uapi/linux/stat.h>.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_chmod(const struct path *path, umode_t mode)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry))))
                return 0;
        return call_int_hook(path_chmod, path, mode);
}

/**
 * security_path_chown() - Check if changing the file's owner/group is allowed
 * @path: file
 * @uid: file owner
 * @gid: file group
 *
 * Check for permission to change owner/group of a file or directory.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry))))
                return 0;
        return call_int_hook(path_chown, path, uid, gid);
}

/**
 * security_path_chroot() - Check if changing the root directory is allowed
 * @path: directory
 *
 * Check for permission to change root directory.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_path_chroot(const struct path *path)
{
        return call_int_hook(path_chroot, path);
}
#endif /* CONFIG_SECURITY_PATH */

/**
 * security_inode_create() - Check if creating a file is allowed
 * @dir: the parent directory
 * @dentry: the file being created
 * @mode: requested file mode
 *
 * Check permission to create a regular file.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_create(struct inode *dir, struct dentry *dentry,
                          umode_t mode)
{
        if (unlikely(IS_PRIVATE(dir)))
                return 0;
        return call_int_hook(inode_create, dir, dentry, mode);
}
EXPORT_SYMBOL_GPL(security_inode_create);

/**
 * security_inode_post_create_tmpfile() - Update inode security of new tmpfile
 * @idmap: idmap of the mount
 * @inode: inode of the new tmpfile
 *
 * Update inode security data after a tmpfile has been created.
 */
void security_inode_post_create_tmpfile(struct mnt_idmap *idmap,
                                        struct inode *inode)
{
        if (unlikely(IS_PRIVATE(inode)))
                return;
        call_void_hook(inode_post_create_tmpfile, idmap, inode);
}

/**
 * security_inode_link() - Check if creating a hard link is allowed
 * @old_dentry: existing file
 * @dir: new parent directory
 * @new_dentry: new link
 *
 * Check permission before creating a new hard link to a file.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_link(struct dentry *old_dentry, struct inode *dir,
                        struct dentry *new_dentry)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry))))
                return 0;
        return call_int_hook(inode_link, old_dentry, dir, new_dentry);
}

/**
 * security_inode_unlink() - Check if removing a hard link is allowed
 * @dir: parent directory
 * @dentry: file
 *
 * Check the permission to remove a hard link to a file.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_unlink(struct inode *dir, struct dentry *dentry)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;
        return call_int_hook(inode_unlink, dir, dentry);
}

/**
 * security_inode_symlink() - Check if creating a symbolic link is allowed
 * @dir: parent directory
 * @dentry: symbolic link
 * @old_name: existing filename
 *
 * Check the permission to create a symbolic link to a file.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_symlink(struct inode *dir, struct dentry *dentry,
                           const char *old_name)
{
        if (unlikely(IS_PRIVATE(dir)))
                return 0;
        return call_int_hook(inode_symlink, dir, dentry, old_name);
}

/**
 * security_inode_mkdir() - Check if creation a new director is allowed
 * @dir: parent directory
 * @dentry: new directory
 * @mode: new directory mode
 *
 * Check permissions to create a new directory in the existing directory
 * associated with inode structure @dir.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
{
        if (unlikely(IS_PRIVATE(dir)))
                return 0;
        return call_int_hook(inode_mkdir, dir, dentry, mode);
}
EXPORT_SYMBOL_GPL(security_inode_mkdir);

/**
 * security_inode_rmdir() - Check if removing a directory is allowed
 * @dir: parent directory
 * @dentry: directory to be removed
 *
 * Check the permission to remove a directory.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_rmdir(struct inode *dir, struct dentry *dentry)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;
        return call_int_hook(inode_rmdir, dir, dentry);
}

/**
 * security_inode_mknod() - Check if creating a special file is allowed
 * @dir: parent directory
 * @dentry: new file
 * @mode: new file mode
 * @dev: device number
 *
 * Check permissions when creating a special file (or a socket or a fifo file
 * created via the mknod system call).  Note that if mknod operation is being
 * done for a regular file, then the create hook will be called and not this
 * hook.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_mknod(struct inode *dir, struct dentry *dentry,
                         umode_t mode, dev_t dev)
{
        if (unlikely(IS_PRIVATE(dir)))
                return 0;
        return call_int_hook(inode_mknod, dir, dentry, mode, dev);
}

/**
 * security_inode_rename() - Check if renaming a file is allowed
 * @old_dir: parent directory of the old file
 * @old_dentry: the old file
 * @new_dir: parent directory of the new file
 * @new_dentry: the new file
 * @flags: flags
 *
 * Check for permission to rename a file or directory.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_rename(struct inode *old_dir, struct dentry *old_dentry,
                          struct inode *new_dir, struct dentry *new_dentry,
                          unsigned int flags)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(old_dentry)) ||
                     (d_is_positive(new_dentry) &&
                      IS_PRIVATE(d_backing_inode(new_dentry)))))
                return 0;

        if (flags & RENAME_EXCHANGE) {
                int err = call_int_hook(inode_rename, new_dir, new_dentry,
                                        old_dir, old_dentry);
                if (err)
                        return err;
        }

        return call_int_hook(inode_rename, old_dir, old_dentry,
                             new_dir, new_dentry);
}

/**
 * security_inode_readlink() - Check if reading a symbolic link is allowed
 * @dentry: link
 *
 * Check the permission to read the symbolic link.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_readlink(struct dentry *dentry)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;
        return call_int_hook(inode_readlink, dentry);
}

/**
 * security_inode_follow_link() - Check if following a symbolic link is allowed
 * @dentry: link dentry
 * @inode: link inode
 * @rcu: true if in RCU-walk mode
 *
 * Check permission to follow a symbolic link when looking up a pathname.  If
 * @rcu is true, @inode is not stable.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_follow_link(struct dentry *dentry, struct inode *inode,
                               bool rcu)
{
        if (unlikely(IS_PRIVATE(inode)))
                return 0;
        return call_int_hook(inode_follow_link, dentry, inode, rcu);
}

/**
 * security_inode_permission() - Check if accessing an inode is allowed
 * @inode: inode
 * @mask: access mask
 *
 * Check permission before accessing an inode.  This hook is called by the
 * existing Linux permission function, so a security module can use it to
 * provide additional checking for existing Linux permission checks.  Notice
 * that this hook is called when a file is opened (as well as many other
 * operations), whereas the file_security_ops permission hook is called when
 * the actual read/write operations are performed.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_permission(struct inode *inode, int mask)
{
        if (unlikely(IS_PRIVATE(inode)))
                return 0;
        return call_int_hook(inode_permission, inode, mask);
}

/**
 * security_inode_setattr() - Check if setting file attributes is allowed
 * @idmap: idmap of the mount
 * @dentry: file
 * @attr: new attributes
 *
 * Check permission before setting file attributes.  Note that the kernel call
 * to notify_change is performed from several locations, whenever file
 * attributes change (such as when a file is truncated, chown/chmod operations,
 * transferring disk quotas, etc).
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_setattr(struct mnt_idmap *idmap,
                           struct dentry *dentry, struct iattr *attr)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;
        return call_int_hook(inode_setattr, idmap, dentry, attr);
}
EXPORT_SYMBOL_GPL(security_inode_setattr);

/**
 * security_inode_post_setattr() - Update the inode after a setattr operation
 * @idmap: idmap of the mount
 * @dentry: file
 * @ia_valid: file attributes set
 *
 * Update inode security field after successful setting file attributes.
 */
void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
                                 int ia_valid)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return;
        call_void_hook(inode_post_setattr, idmap, dentry, ia_valid);
}

/**
 * security_inode_getattr() - Check if getting file attributes is allowed
 * @path: file
 *
 * Check permission before obtaining file attributes.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_getattr(const struct path *path)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(path->dentry))))
                return 0;
        return call_int_hook(inode_getattr, path);
}

/**
 * security_inode_setxattr() - Check if setting file xattrs is allowed
 * @idmap: idmap of the mount
 * @dentry: file
 * @name: xattr name
 * @value: xattr value
 * @size: size of xattr value
 * @flags: flags
 *
 * This hook performs the desired permission checks before setting the extended
 * attributes (xattrs) on @dentry.  It is important to note that we have some
 * additional logic before the main LSM implementation calls to detect if we
 * need to perform an additional capability check at the LSM layer.
 *
 * Normally we enforce a capability check prior to executing the various LSM
 * hook implementations, but if a LSM wants to avoid this capability check,
 * it can register a 'inode_xattr_skipcap' hook and return a value of 1 for
 * xattrs that it wants to avoid the capability check, leaving the LSM fully
 * responsible for enforcing the access control for the specific xattr.  If all
 * of the enabled LSMs refrain from registering a 'inode_xattr_skipcap' hook,
 * or return a 0 (the default return value), the capability check is still
 * performed.  If no 'inode_xattr_skipcap' hooks are registered the capability
 * check is performed.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_setxattr(struct mnt_idmap *idmap,
                            struct dentry *dentry, const char *name,
                            const void *value, size_t size, int flags)
{
        int rc;

        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;

        /* enforce the capability checks at the lsm layer, if needed */
        if (!call_int_hook(inode_xattr_skipcap, name)) {
                rc = cap_inode_setxattr(dentry, name, value, size, flags);
                if (rc)
                        return rc;
        }

        return call_int_hook(inode_setxattr, idmap, dentry, name, value, size,
                             flags);
}

/**
 * security_inode_set_acl() - Check if setting posix acls is allowed
 * @idmap: idmap of the mount
 * @dentry: file
 * @acl_name: acl name
 * @kacl: acl struct
 *
 * Check permission before setting posix acls, the posix acls in @kacl are
 * identified by @acl_name.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_set_acl(struct mnt_idmap *idmap,
                           struct dentry *dentry, const char *acl_name,
                           struct posix_acl *kacl)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;
        return call_int_hook(inode_set_acl, idmap, dentry, acl_name, kacl);
}

/**
 * security_inode_post_set_acl() - Update inode security from posix acls set
 * @dentry: file
 * @acl_name: acl name
 * @kacl: acl struct
 *
 * Update inode security data after successfully setting posix acls on @dentry.
 * The posix acls in @kacl are identified by @acl_name.
 */
void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name,
                                 struct posix_acl *kacl)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return;
        call_void_hook(inode_post_set_acl, dentry, acl_name, kacl);
}

/**
 * security_inode_get_acl() - Check if reading posix acls is allowed
 * @idmap: idmap of the mount
 * @dentry: file
 * @acl_name: acl name
 *
 * Check permission before getting osix acls, the posix acls are identified by
 * @acl_name.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_get_acl(struct mnt_idmap *idmap,
                           struct dentry *dentry, const char *acl_name)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;
        return call_int_hook(inode_get_acl, idmap, dentry, acl_name);
}

/**
 * security_inode_remove_acl() - Check if removing a posix acl is allowed
 * @idmap: idmap of the mount
 * @dentry: file
 * @acl_name: acl name
 *
 * Check permission before removing posix acls, the posix acls are identified
 * by @acl_name.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_remove_acl(struct mnt_idmap *idmap,
                              struct dentry *dentry, const char *acl_name)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;
        return call_int_hook(inode_remove_acl, idmap, dentry, acl_name);
}

/**
 * security_inode_post_remove_acl() - Update inode security after rm posix acls
 * @idmap: idmap of the mount
 * @dentry: file
 * @acl_name: acl name
 *
 * Update inode security data after successfully removing posix acls on
 * @dentry in @idmap. The posix acls are identified by @acl_name.
 */
void security_inode_post_remove_acl(struct mnt_idmap *idmap,
                                    struct dentry *dentry, const char *acl_name)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return;
        call_void_hook(inode_post_remove_acl, idmap, dentry, acl_name);
}

/**
 * security_inode_post_setxattr() - Update the inode after a setxattr operation
 * @dentry: file
 * @name: xattr name
 * @value: xattr value
 * @size: xattr value size
 * @flags: flags
 *
 * Update inode security field after successful setxattr operation.
 */
void security_inode_post_setxattr(struct dentry *dentry, const char *name,
                                  const void *value, size_t size, int flags)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return;
        call_void_hook(inode_post_setxattr, dentry, name, value, size, flags);
}

/**
 * security_inode_getxattr() - Check if xattr access is allowed
 * @dentry: file
 * @name: xattr name
 *
 * Check permission before obtaining the extended attributes identified by
 * @name for @dentry.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_getxattr(struct dentry *dentry, const char *name)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;
        return call_int_hook(inode_getxattr, dentry, name);
}

/**
 * security_inode_listxattr() - Check if listing xattrs is allowed
 * @dentry: file
 *
 * Check permission before obtaining the list of extended attribute names for
 * @dentry.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_listxattr(struct dentry *dentry)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;
        return call_int_hook(inode_listxattr, dentry);
}

/**
 * security_inode_removexattr() - Check if removing an xattr is allowed
 * @idmap: idmap of the mount
 * @dentry: file
 * @name: xattr name
 *
 * This hook performs the desired permission checks before setting the extended
 * attributes (xattrs) on @dentry.  It is important to note that we have some
 * additional logic before the main LSM implementation calls to detect if we
 * need to perform an additional capability check at the LSM layer.
 *
 * Normally we enforce a capability check prior to executing the various LSM
 * hook implementations, but if a LSM wants to avoid this capability check,
 * it can register a 'inode_xattr_skipcap' hook and return a value of 1 for
 * xattrs that it wants to avoid the capability check, leaving the LSM fully
 * responsible for enforcing the access control for the specific xattr.  If all
 * of the enabled LSMs refrain from registering a 'inode_xattr_skipcap' hook,
 * or return a 0 (the default return value), the capability check is still
 * performed.  If no 'inode_xattr_skipcap' hooks are registered the capability
 * check is performed.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inode_removexattr(struct mnt_idmap *idmap,
                               struct dentry *dentry, const char *name)
{
        int rc;

        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return 0;

        /* enforce the capability checks at the lsm layer, if needed */
        if (!call_int_hook(inode_xattr_skipcap, name)) {
                rc = cap_inode_removexattr(idmap, dentry, name);
                if (rc)
                        return rc;
        }

        return call_int_hook(inode_removexattr, idmap, dentry, name);
}

/**
 * security_inode_post_removexattr() - Update the inode after a removexattr op
 * @dentry: file
 * @name: xattr name
 *
 * Update the inode after a successful removexattr operation.
 */
void security_inode_post_removexattr(struct dentry *dentry, const char *name)
{
        if (unlikely(IS_PRIVATE(d_backing_inode(dentry))))
                return;
        call_void_hook(inode_post_removexattr, dentry, name);
}

/**
 * security_inode_need_killpriv() - Check if security_inode_killpriv() required
 * @dentry: associated dentry
 *
 * Called when an inode has been changed to determine if
 * security_inode_killpriv() should be called.
 *
 * Return: Return <0 on error to abort the inode change operation, return 0 if
 *         security_inode_killpriv() does not need to be called, return >0 if
 *         security_inode_killpriv() does need to be called.
 */
int security_inode_need_killpriv(struct dentry *dentry)
{
        return call_int_hook(inode_need_killpriv, dentry);
}

/**
 * security_inode_killpriv() - The setuid bit is removed, update LSM state
 * @idmap: idmap of the mount
 * @dentry: associated dentry
 *
 * The @dentry's setuid bit is being removed.  Remove similar security labels.
 * Called with the dentry->d_inode->i_mutex held.
 *
 * Return: Return 0 on success.  If error is returned, then the operation
 *         causing setuid bit removal is failed.
 */
int security_inode_killpriv(struct mnt_idmap *idmap,
                            struct dentry *dentry)
{
        return call_int_hook(inode_killpriv, idmap, dentry);
}

/**
 * security_inode_getsecurity() - Get the xattr security label of an inode
 * @idmap: idmap of the mount
 * @inode: inode
 * @name: xattr name
 * @buffer: security label buffer
 * @alloc: allocation flag
 *
 * Retrieve a copy of the extended attribute representation of the security
 * label associated with @name for @inode via @buffer.  Note that @name is the
 * remainder of the attribute name after the security prefix has been removed.
 * @alloc is used to specify if the call should return a value via the buffer
 * or just the value length.
 *
 * Return: Returns size of buffer on success.
 */
int security_inode_getsecurity(struct mnt_idmap *idmap,
                               struct inode *inode, const char *name,
                               void **buffer, bool alloc)
{
        if (unlikely(IS_PRIVATE(inode)))
                return LSM_RET_DEFAULT(inode_getsecurity);

        return call_int_hook(inode_getsecurity, idmap, inode, name, buffer,
                             alloc);
}

/**
 * security_inode_setsecurity() - Set the xattr security label of an inode
 * @inode: inode
 * @name: xattr name
 * @value: security label
 * @size: length of security label
 * @flags: flags
 *
 * Set the security label associated with @name for @inode from the extended
 * attribute value @value.  @size indicates the size of the @value in bytes.
 * @flags may be XATTR_CREATE, XATTR_REPLACE, or 0. Note that @name is the
 * remainder of the attribute name after the security. prefix has been removed.
 *
 * Return: Returns 0 on success.
 */
int security_inode_setsecurity(struct inode *inode, const char *name,
                               const void *value, size_t size, int flags)
{
        if (unlikely(IS_PRIVATE(inode)))
                return LSM_RET_DEFAULT(inode_setsecurity);

        return call_int_hook(inode_setsecurity, inode, name, value, size,
                             flags);
}

/**
 * security_inode_listsecurity() - List the xattr security label names
 * @inode: inode
 * @buffer: buffer
 * @buffer_size: size of buffer
 *
 * Copy the extended attribute names for the security labels associated with
 * @inode into @buffer.  The maximum size of @buffer is specified by
 * @buffer_size.  @buffer may be NULL to request the size of the buffer
 * required.
 *
 * Return: Returns number of bytes used/required on success.
 */
int security_inode_listsecurity(struct inode *inode,
                                char *buffer, size_t buffer_size)
{
        if (unlikely(IS_PRIVATE(inode)))
                return 0;
        return call_int_hook(inode_listsecurity, inode, buffer, buffer_size);
}
EXPORT_SYMBOL(security_inode_listsecurity);

/**
 * security_inode_getlsmprop() - Get an inode's LSM data
 * @inode: inode
 * @prop: lsm specific information to return
 *
 * Get the lsm specific information associated with the node.
 */
void security_inode_getlsmprop(struct inode *inode, struct lsm_prop *prop)
{
        call_void_hook(inode_getlsmprop, inode, prop);
}

/**
 * security_inode_copy_up() - Create new creds for an overlayfs copy-up op
 * @src: union dentry of copy-up file
 * @new: newly created creds
 *
 * A file is about to be copied up from lower layer to upper layer of overlay
 * filesystem. Security module can prepare a set of new creds and modify as
 * need be and return new creds. Caller will switch to new creds temporarily to
 * create new file and release newly allocated creds.
 *
 * Return: Returns 0 on success or a negative error code on error.
 */
int security_inode_copy_up(struct dentry *src, struct cred **new)
{
        return call_int_hook(inode_copy_up, src, new);
}
EXPORT_SYMBOL(security_inode_copy_up);

/**
 * security_inode_copy_up_xattr() - Filter xattrs in an overlayfs copy-up op
 * @src: union dentry of copy-up file
 * @name: xattr name
 *
 * Filter the xattrs being copied up when a unioned file is copied up from a
 * lower layer to the union/overlay layer.   The caller is responsible for
 * reading and writing the xattrs, this hook is merely a filter.
 *
 * Return: Returns 0 to accept the xattr, -ECANCELED to discard the xattr,
 *         -EOPNOTSUPP if the security module does not know about attribute,
 *         or a negative error code to abort the copy up.
 */
int security_inode_copy_up_xattr(struct dentry *src, const char *name)
{
        int rc;

        rc = call_int_hook(inode_copy_up_xattr, src, name);
        if (rc != LSM_RET_DEFAULT(inode_copy_up_xattr))
                return rc;

        return LSM_RET_DEFAULT(inode_copy_up_xattr);
}
EXPORT_SYMBOL(security_inode_copy_up_xattr);

/**
 * security_inode_setintegrity() - Set the inode's integrity data
 * @inode: inode
 * @type: type of integrity, e.g. hash digest, signature, etc
 * @value: the integrity value
 * @size: size of the integrity value
 *
 * Register a verified integrity measurement of a inode with LSMs.
 * LSMs should free the previously saved data if @value is NULL.
 *
 * Return: Returns 0 on success, negative values on failure.
 */
int security_inode_setintegrity(const struct inode *inode,
                                enum lsm_integrity_type type, const void *value,
                                size_t size)
{
        return call_int_hook(inode_setintegrity, inode, type, value, size);
}
EXPORT_SYMBOL(security_inode_setintegrity);

/**
 * security_kernfs_init_security() - Init LSM context for a kernfs node
 * @kn_dir: parent kernfs node
 * @kn: the kernfs node to initialize
 *
 * Initialize the security context of a newly created kernfs node based on its
 * own and its parent's attributes.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_kernfs_init_security(struct kernfs_node *kn_dir,
                                  struct kernfs_node *kn)
{
        return call_int_hook(kernfs_init_security, kn_dir, kn);
}

/**
 * security_file_permission() - Check file permissions
 * @file: file
 * @mask: requested permissions
 *
 * Check file permissions before accessing an open file.  This hook is called
 * by various operations that read or write files.  A security module can use
 * this hook to perform additional checking on these operations, e.g. to
 * revalidate permissions on use to support privilege bracketing or policy
 * changes.  Notice that this hook is used when the actual read/write
 * operations are performed, whereas the inode_security_ops hook is called when
 * a file is opened (as well as many other operations).  Although this hook can
 * be used to revalidate permissions for various system call operations that
 * read or write files, it does not address the revalidation of permissions for
 * memory-mapped files.  Security modules must handle this separately if they
 * need such revalidation.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_permission(struct file *file, int mask)
{
        return call_int_hook(file_permission, file, mask);
}

/**
 * security_file_alloc() - Allocate and init a file's LSM blob
 * @file: the file
 *
 * Allocate and attach a security structure to the file->f_security field.  The
 * security field is initialized to NULL when the structure is first created.
 *
 * Return: Return 0 if the hook is successful and permission is granted.
 */
int security_file_alloc(struct file *file)
{
        int rc = lsm_file_alloc(file);

        if (rc)
                return rc;
        rc = call_int_hook(file_alloc_security, file);
        if (unlikely(rc))
                security_file_free(file);
        return rc;
}

/**
 * security_file_release() - Perform actions before releasing the file ref
 * @file: the file
 *
 * Perform actions before releasing the last reference to a file.
 */
void security_file_release(struct file *file)
{
        call_void_hook(file_release, file);
}

/**
 * security_file_free() - Free a file's LSM blob
 * @file: the file
 *
 * Deallocate and free any security structures stored in file->f_security.
 */
void security_file_free(struct file *file)
{
        void *blob;

        call_void_hook(file_free_security, file);

        blob = file->f_security;
        if (blob) {
                file->f_security = NULL;
                kmem_cache_free(lsm_file_cache, blob);
        }
}

/**
 * security_file_ioctl() - Check if an ioctl is allowed
 * @file: associated file
 * @cmd: ioctl cmd
 * @arg: ioctl arguments
 *
 * Check permission for an ioctl operation on @file.  Note that @arg sometimes
 * represents a user space pointer; in other cases, it may be a simple integer
 * value.  When @arg represents a user space pointer, it should never be used
 * by the security module.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
        return call_int_hook(file_ioctl, file, cmd, arg);
}
EXPORT_SYMBOL_GPL(security_file_ioctl);

/**
 * security_file_ioctl_compat() - Check if an ioctl is allowed in compat mode
 * @file: associated file
 * @cmd: ioctl cmd
 * @arg: ioctl arguments
 *
 * Compat version of security_file_ioctl() that correctly handles 32-bit
 * processes running on 64-bit kernels.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_ioctl_compat(struct file *file, unsigned int cmd,
                               unsigned long arg)
{
        return call_int_hook(file_ioctl_compat, file, cmd, arg);
}
EXPORT_SYMBOL_GPL(security_file_ioctl_compat);

static inline unsigned long mmap_prot(struct file *file, unsigned long prot)
{
        /*
         * Does we have PROT_READ and does the application expect
         * it to imply PROT_EXEC?  If not, nothing to talk about...
         */
        if ((prot & (PROT_READ | PROT_EXEC)) != PROT_READ)
                return prot;
        if (!(current->personality & READ_IMPLIES_EXEC))
                return prot;
        /*
         * if that's an anonymous mapping, let it.
         */
        if (!file)
                return prot | PROT_EXEC;
        /*
         * ditto if it's not on noexec mount, except that on !MMU we need
         * NOMMU_MAP_EXEC (== VM_MAYEXEC) in this case
         */
        if (!path_noexec(&file->f_path)) {
#ifndef CONFIG_MMU
                if (file->f_op->mmap_capabilities) {
                        unsigned caps = file->f_op->mmap_capabilities(file);
                        if (!(caps & NOMMU_MAP_EXEC))
                                return prot;
                }
#endif
                return prot | PROT_EXEC;
        }
        /* anything on noexec mount won't get PROT_EXEC */
        return prot;
}

/**
 * security_mmap_file() - Check if mmap'ing a file is allowed
 * @file: file
 * @prot: protection applied by the kernel
 * @flags: flags
 *
 * Check permissions for a mmap operation.  The @file may be NULL, e.g. if
 * mapping anonymous memory.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_mmap_file(struct file *file, unsigned long prot,
                       unsigned long flags)
{
        return call_int_hook(mmap_file, file, prot, mmap_prot(file, prot),
                             flags);
}

/**
 * security_mmap_addr() - Check if mmap'ing an address is allowed
 * @addr: address
 *
 * Check permissions for a mmap operation at @addr.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_mmap_addr(unsigned long addr)
{
        return call_int_hook(mmap_addr, addr);
}

/**
 * security_file_mprotect() - Check if changing memory protections is allowed
 * @vma: memory region
 * @reqprot: application requested protection
 * @prot: protection applied by the kernel
 *
 * Check permissions before changing memory access permissions.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot,
                           unsigned long prot)
{
        return call_int_hook(file_mprotect, vma, reqprot, prot);
}

/**
 * security_file_lock() - Check if a file lock is allowed
 * @file: file
 * @cmd: lock operation (e.g. F_RDLCK, F_WRLCK)
 *
 * Check permission before performing file locking operations.  Note the hook
 * mediates both flock and fcntl style locks.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_lock(struct file *file, unsigned int cmd)
{
        return call_int_hook(file_lock, file, cmd);
}

/**
 * security_file_fcntl() - Check if fcntl() op is allowed
 * @file: file
 * @cmd: fcntl command
 * @arg: command argument
 *
 * Check permission before allowing the file operation specified by @cmd from
 * being performed on the file @file.  Note that @arg sometimes represents a
 * user space pointer; in other cases, it may be a simple integer value.  When
 * @arg represents a user space pointer, it should never be used by the
 * security module.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
{
        return call_int_hook(file_fcntl, file, cmd, arg);
}

/**
 * security_file_set_fowner() - Set the file owner info in the LSM blob
 * @file: the file
 *
 * Save owner security information (typically from current->security) in
 * file->f_security for later use by the send_sigiotask hook.
 *
 * This hook is called with file->f_owner.lock held.
 *
 * Return: Returns 0 on success.
 */
void security_file_set_fowner(struct file *file)
{
        call_void_hook(file_set_fowner, file);
}

/**
 * security_file_send_sigiotask() - Check if sending SIGIO/SIGURG is allowed
 * @tsk: target task
 * @fown: signal sender
 * @sig: signal to be sent, SIGIO is sent if 0
 *
 * Check permission for the file owner @fown to send SIGIO or SIGURG to the
 * process @tsk.  Note that this hook is sometimes called from interrupt.  Note
 * that the fown_struct, @fown, is never outside the context of a struct file,
 * so the file structure (and associated security information) can always be
 * obtained: container_of(fown, struct file, f_owner).
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_send_sigiotask(struct task_struct *tsk,
                                 struct fown_struct *fown, int sig)
{
        return call_int_hook(file_send_sigiotask, tsk, fown, sig);
}

/**
 * security_file_receive() - Check if receiving a file via IPC is allowed
 * @file: file being received
 *
 * This hook allows security modules to control the ability of a process to
 * receive an open file descriptor via socket IPC.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_receive(struct file *file)
{
        return call_int_hook(file_receive, file);
}

/**
 * security_file_open() - Save open() time state for late use by the LSM
 * @file:
 *
 * Save open-time permission checking state for later use upon file_permission,
 * and recheck access if anything has changed since inode_permission.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_open(struct file *file)
{
        return call_int_hook(file_open, file);
}

/**
 * security_file_post_open() - Evaluate a file after it has been opened
 * @file: the file
 * @mask: access mask
 *
 * Evaluate an opened file and the access mask requested with open(). The hook
 * is useful for LSMs that require the file content to be available in order to
 * make decisions.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_post_open(struct file *file, int mask)
{
        return call_int_hook(file_post_open, file, mask);
}
EXPORT_SYMBOL_GPL(security_file_post_open);

/**
 * security_file_truncate() - Check if truncating a file is allowed
 * @file: file
 *
 * Check permission before truncating a file, i.e. using ftruncate.  Note that
 * truncation permission may also be checked based on the path, using the
 * @path_truncate hook.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_file_truncate(struct file *file)
{
        return call_int_hook(file_truncate, file);
}

/**
 * security_task_alloc() - Allocate a task's LSM blob
 * @task: the task
 * @clone_flags: flags indicating what is being shared
 *
 * Handle allocation of task-related resources.
 *
 * Return: Returns a zero on success, negative values on failure.
 */
int security_task_alloc(struct task_struct *task, unsigned long clone_flags)
{
        int rc = lsm_task_alloc(task);

        if (rc)
                return rc;
        rc = call_int_hook(task_alloc, task, clone_flags);
        if (unlikely(rc))
                security_task_free(task);
        return rc;
}

/**
 * security_task_free() - Free a task's LSM blob and related resources
 * @task: task
 *
 * Handle release of task-related resources.  Note that this can be called from
 * interrupt context.
 */
void security_task_free(struct task_struct *task)
{
        call_void_hook(task_free, task);

        kfree(task->security);
        task->security = NULL;
}

/**
 * security_cred_alloc_blank() - Allocate the min memory to allow cred_transfer
 * @cred: credentials
 * @gfp: gfp flags
 *
 * Only allocate sufficient memory and attach to @cred such that
 * cred_transfer() will not get ENOMEM.
 *
 * Return: Returns 0 on success, negative values on failure.
 */
int security_cred_alloc_blank(struct cred *cred, gfp_t gfp)
{
        int rc = lsm_cred_alloc(cred, gfp);

        if (rc)
                return rc;

        rc = call_int_hook(cred_alloc_blank, cred, gfp);
        if (unlikely(rc))
                security_cred_free(cred);
        return rc;
}

/**
 * security_cred_free() - Free the cred's LSM blob and associated resources
 * @cred: credentials
 *
 * Deallocate and clear the cred->security field in a set of credentials.
 */
void security_cred_free(struct cred *cred)
{
        /*
         * There is a failure case in prepare_creds() that
         * may result in a call here with ->security being NULL.
         */
        if (unlikely(cred->security == NULL))
                return;

        call_void_hook(cred_free, cred);

        kfree(cred->security);
        cred->security = NULL;
}

/**
 * security_prepare_creds() - Prepare a new set of credentials
 * @new: new credentials
 * @old: original credentials
 * @gfp: gfp flags
 *
 * Prepare a new set of credentials by copying the data from the old set.
 *
 * Return: Returns 0 on success, negative values on failure.
 */
int security_prepare_creds(struct cred *new, const struct cred *old, gfp_t gfp)
{
        int rc = lsm_cred_alloc(new, gfp);

        if (rc)
                return rc;

        rc = call_int_hook(cred_prepare, new, old, gfp);
        if (unlikely(rc))
                security_cred_free(new);
        return rc;
}

/**
 * security_transfer_creds() - Transfer creds
 * @new: target credentials
 * @old: original credentials
 *
 * Transfer data from original creds to new creds.
 */
void security_transfer_creds(struct cred *new, const struct cred *old)
{
        call_void_hook(cred_transfer, new, old);
}

/**
 * security_cred_getsecid() - Get the secid from a set of credentials
 * @c: credentials
 * @secid: secid value
 *
 * Retrieve the security identifier of the cred structure @c.  In case of
 * failure, @secid will be set to zero.
 */
void security_cred_getsecid(const struct cred *c, u32 *secid)
{
        *secid = 0;
        call_void_hook(cred_getsecid, c, secid);
}
EXPORT_SYMBOL(security_cred_getsecid);

/**
 * security_cred_getlsmprop() - Get the LSM data from a set of credentials
 * @c: credentials
 * @prop: destination for the LSM data
 *
 * Retrieve the security data of the cred structure @c.  In case of
 * failure, @prop will be cleared.
 */
void security_cred_getlsmprop(const struct cred *c, struct lsm_prop *prop)
{
        lsmprop_init(prop);
        call_void_hook(cred_getlsmprop, c, prop);
}
EXPORT_SYMBOL(security_cred_getlsmprop);

/**
 * security_kernel_act_as() - Set the kernel credentials to act as secid
 * @new: credentials
 * @secid: secid
 *
 * Set the credentials for a kernel service to act as (subjective context).
 * The current task must be the one that nominated @secid.
 *
 * Return: Returns 0 if successful.
 */
int security_kernel_act_as(struct cred *new, u32 secid)
{
        return call_int_hook(kernel_act_as, new, secid);
}

/**
 * security_kernel_create_files_as() - Set file creation context using an inode
 * @new: target credentials
 * @inode: reference inode
 *
 * Set the file creation context in a set of credentials to be the same as the
 * objective context of the specified inode.  The current task must be the one
 * that nominated @inode.
 *
 * Return: Returns 0 if successful.
 */
int security_kernel_create_files_as(struct cred *new, struct inode *inode)
{
        return call_int_hook(kernel_create_files_as, new, inode);
}

/**
 * security_kernel_module_request() - Check if loading a module is allowed
 * @kmod_name: module name
 *
 * Ability to trigger the kernel to automatically upcall to userspace for
 * userspace to load a kernel module with the given name.
 *
 * Return: Returns 0 if successful.
 */
int security_kernel_module_request(char *kmod_name)
{
        return call_int_hook(kernel_module_request, kmod_name);
}

/**
 * security_kernel_read_file() - Read a file specified by userspace
 * @file: file
 * @id: file identifier
 * @contents: trust if security_kernel_post_read_file() will be called
 *
 * Read a file specified by userspace.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_kernel_read_file(struct file *file, enum kernel_read_file_id id,
                              bool contents)
{
        return call_int_hook(kernel_read_file, file, id, contents);
}
EXPORT_SYMBOL_GPL(security_kernel_read_file);

/**
 * security_kernel_post_read_file() - Read a file specified by userspace
 * @file: file
 * @buf: file contents
 * @size: size of file contents
 * @id: file identifier
 *
 * Read a file specified by userspace.  This must be paired with a prior call
 * to security_kernel_read_file() call that indicated this hook would also be
 * called, see security_kernel_read_file() for more information.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_kernel_post_read_file(struct file *file, char *buf, loff_t size,
                                   enum kernel_read_file_id id)
{
        return call_int_hook(kernel_post_read_file, file, buf, size, id);
}
EXPORT_SYMBOL_GPL(security_kernel_post_read_file);

/**
 * security_kernel_load_data() - Load data provided by userspace
 * @id: data identifier
 * @contents: true if security_kernel_post_load_data() will be called
 *
 * Load data provided by userspace.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_kernel_load_data(enum kernel_load_data_id id, bool contents)
{
        return call_int_hook(kernel_load_data, id, contents);
}
EXPORT_SYMBOL_GPL(security_kernel_load_data);

/**
 * security_kernel_post_load_data() - Load userspace data from a non-file source
 * @buf: data
 * @size: size of data
 * @id: data identifier
 * @description: text description of data, specific to the id value
 *
 * Load data provided by a non-file source (usually userspace buffer).  This
 * must be paired with a prior security_kernel_load_data() call that indicated
 * this hook would also be called, see security_kernel_load_data() for more
 * information.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_kernel_post_load_data(char *buf, loff_t size,
                                   enum kernel_load_data_id id,
                                   char *description)
{
        return call_int_hook(kernel_post_load_data, buf, size, id, description);
}
EXPORT_SYMBOL_GPL(security_kernel_post_load_data);

/**
 * security_task_fix_setuid() - Update LSM with new user id attributes
 * @new: updated credentials
 * @old: credentials being replaced
 * @flags: LSM_SETID_* flag values
 *
 * Update the module's state after setting one or more of the user identity
 * attributes of the current process.  The @flags parameter indicates which of
 * the set*uid system calls invoked this hook.  If @new is the set of
 * credentials that will be installed.  Modifications should be made to this
 * rather than to @current->cred.
 *
 * Return: Returns 0 on success.
 */
int security_task_fix_setuid(struct cred *new, const struct cred *old,
                             int flags)
{
        return call_int_hook(task_fix_setuid, new, old, flags);
}

/**
 * security_task_fix_setgid() - Update LSM with new group id attributes
 * @new: updated credentials
 * @old: credentials being replaced
 * @flags: LSM_SETID_* flag value
 *
 * Update the module's state after setting one or more of the group identity
 * attributes of the current process.  The @flags parameter indicates which of
 * the set*gid system calls invoked this hook.  @new is the set of credentials
 * that will be installed.  Modifications should be made to this rather than to
 * @current->cred.
 *
 * Return: Returns 0 on success.
 */
int security_task_fix_setgid(struct cred *new, const struct cred *old,
                             int flags)
{
        return call_int_hook(task_fix_setgid, new, old, flags);
}

/**
 * security_task_fix_setgroups() - Update LSM with new supplementary groups
 * @new: updated credentials
 * @old: credentials being replaced
 *
 * Update the module's state after setting the supplementary group identity
 * attributes of the current process.  @new is the set of credentials that will
 * be installed.  Modifications should be made to this rather than to
 * @current->cred.
 *
 * Return: Returns 0 on success.
 */
int security_task_fix_setgroups(struct cred *new, const struct cred *old)
{
        return call_int_hook(task_fix_setgroups, new, old);
}

/**
 * security_task_setpgid() - Check if setting the pgid is allowed
 * @p: task being modified
 * @pgid: new pgid
 *
 * Check permission before setting the process group identifier of the process
 * @p to @pgid.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_setpgid(struct task_struct *p, pid_t pgid)
{
        return call_int_hook(task_setpgid, p, pgid);
}

/**
 * security_task_getpgid() - Check if getting the pgid is allowed
 * @p: task
 *
 * Check permission before getting the process group identifier of the process
 * @p.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_getpgid(struct task_struct *p)
{
        return call_int_hook(task_getpgid, p);
}

/**
 * security_task_getsid() - Check if getting the session id is allowed
 * @p: task
 *
 * Check permission before getting the session identifier of the process @p.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_getsid(struct task_struct *p)
{
        return call_int_hook(task_getsid, p);
}

/**
 * security_current_getlsmprop_subj() - Current task's subjective LSM data
 * @prop: lsm specific information
 *
 * Retrieve the subjective security identifier of the current task and return
 * it in @prop.
 */
void security_current_getlsmprop_subj(struct lsm_prop *prop)
{
        lsmprop_init(prop);
        call_void_hook(current_getlsmprop_subj, prop);
}
EXPORT_SYMBOL(security_current_getlsmprop_subj);

/**
 * security_task_getlsmprop_obj() - Get a task's objective LSM data
 * @p: target task
 * @prop: lsm specific information
 *
 * Retrieve the objective security identifier of the task_struct in @p and
 * return it in @prop.
 */
void security_task_getlsmprop_obj(struct task_struct *p, struct lsm_prop *prop)
{
        lsmprop_init(prop);
        call_void_hook(task_getlsmprop_obj, p, prop);
}
EXPORT_SYMBOL(security_task_getlsmprop_obj);

/**
 * security_task_setnice() - Check if setting a task's nice value is allowed
 * @p: target task
 * @nice: nice value
 *
 * Check permission before setting the nice value of @p to @nice.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_setnice(struct task_struct *p, int nice)
{
        return call_int_hook(task_setnice, p, nice);
}

/**
 * security_task_setioprio() - Check if setting a task's ioprio is allowed
 * @p: target task
 * @ioprio: ioprio value
 *
 * Check permission before setting the ioprio value of @p to @ioprio.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_setioprio(struct task_struct *p, int ioprio)
{
        return call_int_hook(task_setioprio, p, ioprio);
}

/**
 * security_task_getioprio() - Check if getting a task's ioprio is allowed
 * @p: task
 *
 * Check permission before getting the ioprio value of @p.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_getioprio(struct task_struct *p)
{
        return call_int_hook(task_getioprio, p);
}

/**
 * security_task_prlimit() - Check if get/setting resources limits is allowed
 * @cred: current task credentials
 * @tcred: target task credentials
 * @flags: LSM_PRLIMIT_* flag bits indicating a get/set/both
 *
 * Check permission before getting and/or setting the resource limits of
 * another task.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_prlimit(const struct cred *cred, const struct cred *tcred,
                          unsigned int flags)
{
        return call_int_hook(task_prlimit, cred, tcred, flags);
}

/**
 * security_task_setrlimit() - Check if setting a new rlimit value is allowed
 * @p: target task's group leader
 * @resource: resource whose limit is being set
 * @new_rlim: new resource limit
 *
 * Check permission before setting the resource limits of process @p for
 * @resource to @new_rlim.  The old resource limit values can be examined by
 * dereferencing (p->signal->rlim + resource).
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_setrlimit(struct task_struct *p, unsigned int resource,
                            struct rlimit *new_rlim)
{
        return call_int_hook(task_setrlimit, p, resource, new_rlim);
}

/**
 * security_task_setscheduler() - Check if setting sched policy/param is allowed
 * @p: target task
 *
 * Check permission before setting scheduling policy and/or parameters of
 * process @p.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_setscheduler(struct task_struct *p)
{
        return call_int_hook(task_setscheduler, p);
}

/**
 * security_task_getscheduler() - Check if getting scheduling info is allowed
 * @p: target task
 *
 * Check permission before obtaining scheduling information for process @p.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_getscheduler(struct task_struct *p)
{
        return call_int_hook(task_getscheduler, p);
}

/**
 * security_task_movememory() - Check if moving memory is allowed
 * @p: task
 *
 * Check permission before moving memory owned by process @p.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_movememory(struct task_struct *p)
{
        return call_int_hook(task_movememory, p);
}

/**
 * security_task_kill() - Check if sending a signal is allowed
 * @p: target process
 * @info: signal information
 * @sig: signal value
 * @cred: credentials of the signal sender, NULL if @current
 *
 * Check permission before sending signal @sig to @p.  @info can be NULL, the
 * constant 1, or a pointer to a kernel_siginfo structure.  If @info is 1 or
 * SI_FROMKERNEL(info) is true, then the signal should be viewed as coming from
 * the kernel and should typically be permitted.  SIGIO signals are handled
 * separately by the send_sigiotask hook in file_security_ops.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_task_kill(struct task_struct *p, struct kernel_siginfo *info,
                       int sig, const struct cred *cred)
{
        return call_int_hook(task_kill, p, info, sig, cred);
}

/**
 * security_task_prctl() - Check if a prctl op is allowed
 * @option: operation
 * @arg2: argument
 * @arg3: argument
 * @arg4: argument
 * @arg5: argument
 *
 * Check permission before performing a process control operation on the
 * current process.
 *
 * Return: Return -ENOSYS if no-one wanted to handle this op, any other value
 *         to cause prctl() to return immediately with that value.
 */
int security_task_prctl(int option, unsigned long arg2, unsigned long arg3,
                        unsigned long arg4, unsigned long arg5)
{
        int thisrc;
        int rc = LSM_RET_DEFAULT(task_prctl);
        struct lsm_static_call *scall;

        lsm_for_each_hook(scall, task_prctl) {
                thisrc = scall->hl->hook.task_prctl(option, arg2, arg3, arg4, arg5);
                if (thisrc != LSM_RET_DEFAULT(task_prctl)) {
                        rc = thisrc;
                        if (thisrc != 0)
                                break;
                }
        }
        return rc;
}

/**
 * security_task_to_inode() - Set the security attributes of a task's inode
 * @p: task
 * @inode: inode
 *
 * Set the security attributes for an inode based on an associated task's
 * security attributes, e.g. for /proc/pid inodes.
 */
void security_task_to_inode(struct task_struct *p, struct inode *inode)
{
        call_void_hook(task_to_inode, p, inode);
}

/**
 * security_create_user_ns() - Check if creating a new userns is allowed
 * @cred: prepared creds
 *
 * Check permission prior to creating a new user namespace.
 *
 * Return: Returns 0 if successful, otherwise < 0 error code.
 */
int security_create_user_ns(const struct cred *cred)
{
        return call_int_hook(userns_create, cred);
}

/**
 * security_ipc_permission() - Check if sysv ipc access is allowed
 * @ipcp: ipc permission structure
 * @flag: requested permissions
 *
 * Check permissions for access to IPC.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag)
{
        return call_int_hook(ipc_permission, ipcp, flag);
}

/**
 * security_ipc_getlsmprop() - Get the sysv ipc object LSM data
 * @ipcp: ipc permission structure
 * @prop: pointer to lsm information
 *
 * Get the lsm information associated with the ipc object.
 */

void security_ipc_getlsmprop(struct kern_ipc_perm *ipcp, struct lsm_prop *prop)
{
        lsmprop_init(prop);
        call_void_hook(ipc_getlsmprop, ipcp, prop);
}

/**
 * security_msg_msg_alloc() - Allocate a sysv ipc message LSM blob
 * @msg: message structure
 *
 * Allocate and attach a security structure to the msg->security field.  The
 * security field is initialized to NULL when the structure is first created.
 *
 * Return: Return 0 if operation was successful and permission is granted.
 */
int security_msg_msg_alloc(struct msg_msg *msg)
{
        int rc = lsm_msg_msg_alloc(msg);

        if (unlikely(rc))
                return rc;
        rc = call_int_hook(msg_msg_alloc_security, msg);
        if (unlikely(rc))
                security_msg_msg_free(msg);
        return rc;
}

/**
 * security_msg_msg_free() - Free a sysv ipc message LSM blob
 * @msg: message structure
 *
 * Deallocate the security structure for this message.
 */
void security_msg_msg_free(struct msg_msg *msg)
{
        call_void_hook(msg_msg_free_security, msg);
        kfree(msg->security);
        msg->security = NULL;
}

/**
 * security_msg_queue_alloc() - Allocate a sysv ipc msg queue LSM blob
 * @msq: sysv ipc permission structure
 *
 * Allocate and attach a security structure to @msg. The security field is
 * initialized to NULL when the structure is first created.
 *
 * Return: Returns 0 if operation was successful and permission is granted.
 */
int security_msg_queue_alloc(struct kern_ipc_perm *msq)
{
        int rc = lsm_ipc_alloc(msq);

        if (unlikely(rc))
                return rc;
        rc = call_int_hook(msg_queue_alloc_security, msq);
        if (unlikely(rc))
                security_msg_queue_free(msq);
        return rc;
}

/**
 * security_msg_queue_free() - Free a sysv ipc msg queue LSM blob
 * @msq: sysv ipc permission structure
 *
 * Deallocate security field @perm->security for the message queue.
 */
void security_msg_queue_free(struct kern_ipc_perm *msq)
{
        call_void_hook(msg_queue_free_security, msq);
        kfree(msq->security);
        msq->security = NULL;
}

/**
 * security_msg_queue_associate() - Check if a msg queue operation is allowed
 * @msq: sysv ipc permission structure
 * @msqflg: operation flags
 *
 * Check permission when a message queue is requested through the msgget system
 * call. This hook is only called when returning the message queue identifier
 * for an existing message queue, not when a new message queue is created.
 *
 * Return: Return 0 if permission is granted.
 */
int security_msg_queue_associate(struct kern_ipc_perm *msq, int msqflg)
{
        return call_int_hook(msg_queue_associate, msq, msqflg);
}

/**
 * security_msg_queue_msgctl() - Check if a msg queue operation is allowed
 * @msq: sysv ipc permission structure
 * @cmd: operation
 *
 * Check permission when a message control operation specified by @cmd is to be
 * performed on the message queue with permissions.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_msg_queue_msgctl(struct kern_ipc_perm *msq, int cmd)
{
        return call_int_hook(msg_queue_msgctl, msq, cmd);
}

/**
 * security_msg_queue_msgsnd() - Check if sending a sysv ipc message is allowed
 * @msq: sysv ipc permission structure
 * @msg: message
 * @msqflg: operation flags
 *
 * Check permission before a message, @msg, is enqueued on the message queue
 * with permissions specified in @msq.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_msg_queue_msgsnd(struct kern_ipc_perm *msq,
                              struct msg_msg *msg, int msqflg)
{
        return call_int_hook(msg_queue_msgsnd, msq, msg, msqflg);
}

/**
 * security_msg_queue_msgrcv() - Check if receiving a sysv ipc msg is allowed
 * @msq: sysv ipc permission structure
 * @msg: message
 * @target: target task
 * @type: type of message requested
 * @mode: operation flags
 *
 * Check permission before a message, @msg, is removed from the message        queue.
 * The @target task structure contains a pointer to the process that will be
 * receiving the message (not equal to the current process when inline receives
 * are being performed).
 *
 * Return: Returns 0 if permission is granted.
 */
int security_msg_queue_msgrcv(struct kern_ipc_perm *msq, struct msg_msg *msg,
                              struct task_struct *target, long type, int mode)
{
        return call_int_hook(msg_queue_msgrcv, msq, msg, target, type, mode);
}

/**
 * security_shm_alloc() - Allocate a sysv shm LSM blob
 * @shp: sysv ipc permission structure
 *
 * Allocate and attach a security structure to the @shp security field.  The
 * security field is initialized to NULL when the structure is first created.
 *
 * Return: Returns 0 if operation was successful and permission is granted.
 */
int security_shm_alloc(struct kern_ipc_perm *shp)
{
        int rc = lsm_ipc_alloc(shp);

        if (unlikely(rc))
                return rc;
        rc = call_int_hook(shm_alloc_security, shp);
        if (unlikely(rc))
                security_shm_free(shp);
        return rc;
}

/**
 * security_shm_free() - Free a sysv shm LSM blob
 * @shp: sysv ipc permission structure
 *
 * Deallocate the security structure @perm->security for the memory segment.
 */
void security_shm_free(struct kern_ipc_perm *shp)
{
        call_void_hook(shm_free_security, shp);
        kfree(shp->security);
        shp->security = NULL;
}

/**
 * security_shm_associate() - Check if a sysv shm operation is allowed
 * @shp: sysv ipc permission structure
 * @shmflg: operation flags
 *
 * Check permission when a shared memory region is requested through the shmget
 * system call. This hook is only called when returning the shared memory
 * region identifier for an existing region, not when a new shared memory
 * region is created.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_shm_associate(struct kern_ipc_perm *shp, int shmflg)
{
        return call_int_hook(shm_associate, shp, shmflg);
}

/**
 * security_shm_shmctl() - Check if a sysv shm operation is allowed
 * @shp: sysv ipc permission structure
 * @cmd: operation
 *
 * Check permission when a shared memory control operation specified by @cmd is
 * to be performed on the shared memory region with permissions in @shp.
 *
 * Return: Return 0 if permission is granted.
 */
int security_shm_shmctl(struct kern_ipc_perm *shp, int cmd)
{
        return call_int_hook(shm_shmctl, shp, cmd);
}

/**
 * security_shm_shmat() - Check if a sysv shm attach operation is allowed
 * @shp: sysv ipc permission structure
 * @shmaddr: address of memory region to attach
 * @shmflg: operation flags
 *
 * Check permissions prior to allowing the shmat system call to attach the
 * shared memory segment with permissions @shp to the data segment of the
 * calling process. The attaching address is specified by @shmaddr.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_shm_shmat(struct kern_ipc_perm *shp,
                       char __user *shmaddr, int shmflg)
{
        return call_int_hook(shm_shmat, shp, shmaddr, shmflg);
}

/**
 * security_sem_alloc() - Allocate a sysv semaphore LSM blob
 * @sma: sysv ipc permission structure
 *
 * Allocate and attach a security structure to the @sma security field. The
 * security field is initialized to NULL when the structure is first created.
 *
 * Return: Returns 0 if operation was successful and permission is granted.
 */
int security_sem_alloc(struct kern_ipc_perm *sma)
{
        int rc = lsm_ipc_alloc(sma);

        if (unlikely(rc))
                return rc;
        rc = call_int_hook(sem_alloc_security, sma);
        if (unlikely(rc))
                security_sem_free(sma);
        return rc;
}

/**
 * security_sem_free() - Free a sysv semaphore LSM blob
 * @sma: sysv ipc permission structure
 *
 * Deallocate security structure @sma->security for the semaphore.
 */
void security_sem_free(struct kern_ipc_perm *sma)
{
        call_void_hook(sem_free_security, sma);
        kfree(sma->security);
        sma->security = NULL;
}

/**
 * security_sem_associate() - Check if a sysv semaphore operation is allowed
 * @sma: sysv ipc permission structure
 * @semflg: operation flags
 *
 * Check permission when a semaphore is requested through the semget system
 * call. This hook is only called when returning the semaphore identifier for
 * an existing semaphore, not when a new one must be created.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sem_associate(struct kern_ipc_perm *sma, int semflg)
{
        return call_int_hook(sem_associate, sma, semflg);
}

/**
 * security_sem_semctl() - Check if a sysv semaphore operation is allowed
 * @sma: sysv ipc permission structure
 * @cmd: operation
 *
 * Check permission when a semaphore operation specified by @cmd is to be
 * performed on the semaphore.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sem_semctl(struct kern_ipc_perm *sma, int cmd)
{
        return call_int_hook(sem_semctl, sma, cmd);
}

/**
 * security_sem_semop() - Check if a sysv semaphore operation is allowed
 * @sma: sysv ipc permission structure
 * @sops: operations to perform
 * @nsops: number of operations
 * @alter: flag indicating changes will be made
 *
 * Check permissions before performing operations on members of the semaphore
 * set. If the @alter flag is nonzero, the semaphore set may be modified.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sem_semop(struct kern_ipc_perm *sma, struct sembuf *sops,
                       unsigned nsops, int alter)
{
        return call_int_hook(sem_semop, sma, sops, nsops, alter);
}

/**
 * security_d_instantiate() - Populate an inode's LSM state based on a dentry
 * @dentry: dentry
 * @inode: inode
 *
 * Fill in @inode security information for a @dentry if allowed.
 */
void security_d_instantiate(struct dentry *dentry, struct inode *inode)
{
        if (unlikely(inode && IS_PRIVATE(inode)))
                return;
        call_void_hook(d_instantiate, dentry, inode);
}
EXPORT_SYMBOL(security_d_instantiate);

/*
 * Please keep this in sync with it's counterpart in security/lsm_syscalls.c
 */

/**
 * security_getselfattr - Read an LSM attribute of the current process.
 * @attr: which attribute to return
 * @uctx: the user-space destination for the information, or NULL
 * @size: pointer to the size of space available to receive the data
 * @flags: special handling options. LSM_FLAG_SINGLE indicates that only
 * attributes associated with the LSM identified in the passed @ctx be
 * reported.
 *
 * A NULL value for @uctx can be used to get both the number of attributes
 * and the size of the data.
 *
 * Returns the number of attributes found on success, negative value
 * on error. @size is reset to the total size of the data.
 * If @size is insufficient to contain the data -E2BIG is returned.
 */
int security_getselfattr(unsigned int attr, struct lsm_ctx __user *uctx,
                         u32 __user *size, u32 flags)
{
        struct lsm_static_call *scall;
        struct lsm_ctx lctx = { .id = LSM_ID_UNDEF, };
        u8 __user *base = (u8 __user *)uctx;
        u32 entrysize;
        u32 total = 0;
        u32 left;
        bool toobig = false;
        bool single = false;
        int count = 0;
        int rc;

        if (attr == LSM_ATTR_UNDEF)
                return -EINVAL;
        if (size == NULL)
                return -EINVAL;
        if (get_user(left, size))
                return -EFAULT;

        if (flags) {
                /*
                 * Only flag supported is LSM_FLAG_SINGLE
                 */
                if (flags != LSM_FLAG_SINGLE || !uctx)
                        return -EINVAL;
                if (copy_from_user(&lctx, uctx, sizeof(lctx)))
                        return -EFAULT;
                /*
                 * If the LSM ID isn't specified it is an error.
                 */
                if (lctx.id == LSM_ID_UNDEF)
                        return -EINVAL;
                single = true;
        }

        /*
         * In the usual case gather all the data from the LSMs.
         * In the single case only get the data from the LSM specified.
         */
        lsm_for_each_hook(scall, getselfattr) {
                if (single && lctx.id != scall->hl->lsmid->id)
                        continue;
                entrysize = left;
                if (base)
                        uctx = (struct lsm_ctx __user *)(base + total);
                rc = scall->hl->hook.getselfattr(attr, uctx, &entrysize, flags);
                if (rc == -EOPNOTSUPP) {
                        rc = 0;
                        continue;
                }
                if (rc == -E2BIG) {
                        rc = 0;
                        left = 0;
                        toobig = true;
                } else if (rc < 0)
                        return rc;
                else
                        left -= entrysize;

                total += entrysize;
                count += rc;
                if (single)
                        break;
        }
        if (put_user(total, size))
                return -EFAULT;
        if (toobig)
                return -E2BIG;
        if (count == 0)
                return LSM_RET_DEFAULT(getselfattr);
        return count;
}

/*
 * Please keep this in sync with it's counterpart in security/lsm_syscalls.c
 */

/**
 * security_setselfattr - Set an LSM attribute on the current process.
 * @attr: which attribute to set
 * @uctx: the user-space source for the information
 * @size: the size of the data
 * @flags: reserved for future use, must be 0
 *
 * Set an LSM attribute for the current process. The LSM, attribute
 * and new value are included in @uctx.
 *
 * Returns 0 on success, -EINVAL if the input is inconsistent, -EFAULT
 * if the user buffer is inaccessible, E2BIG if size is too big, or an
 * LSM specific failure.
 */
int security_setselfattr(unsigned int attr, struct lsm_ctx __user *uctx,
                         u32 size, u32 flags)
{
        struct lsm_static_call *scall;
        struct lsm_ctx *lctx;
        int rc = LSM_RET_DEFAULT(setselfattr);
        u64 required_len;

        if (flags)
                return -EINVAL;
        if (size < sizeof(*lctx))
                return -EINVAL;
        if (size > PAGE_SIZE)
                return -E2BIG;

        lctx = memdup_user(uctx, size);
        if (IS_ERR(lctx))
                return PTR_ERR(lctx);

        if (size < lctx->len ||
            check_add_overflow(sizeof(*lctx), lctx->ctx_len, &required_len) ||
            lctx->len < required_len) {
                rc = -EINVAL;
                goto free_out;
        }

        lsm_for_each_hook(scall, setselfattr)
                if ((scall->hl->lsmid->id) == lctx->id) {
                        rc = scall->hl->hook.setselfattr(attr, lctx, size, flags);
                        break;
                }

free_out:
        kfree(lctx);
        return rc;
}

/**
 * security_getprocattr() - Read an attribute for a task
 * @p: the task
 * @lsmid: LSM identification
 * @name: attribute name
 * @value: attribute value
 *
 * Read attribute @name for task @p and store it into @value if allowed.
 *
 * Return: Returns the length of @value on success, a negative value otherwise.
 */
int security_getprocattr(struct task_struct *p, int lsmid, const char *name,
                         char **value)
{
        struct lsm_static_call *scall;

        lsm_for_each_hook(scall, getprocattr) {
                if (lsmid != 0 && lsmid != scall->hl->lsmid->id)
                        continue;
                return scall->hl->hook.getprocattr(p, name, value);
        }
        return LSM_RET_DEFAULT(getprocattr);
}

/**
 * security_setprocattr() - Set an attribute for a task
 * @lsmid: LSM identification
 * @name: attribute name
 * @value: attribute value
 * @size: attribute value size
 *
 * Write (set) the current task's attribute @name to @value, size @size if
 * allowed.
 *
 * Return: Returns bytes written on success, a negative value otherwise.
 */
int security_setprocattr(int lsmid, const char *name, void *value, size_t size)
{
        struct lsm_static_call *scall;

        lsm_for_each_hook(scall, setprocattr) {
                if (lsmid != 0 && lsmid != scall->hl->lsmid->id)
                        continue;
                return scall->hl->hook.setprocattr(name, value, size);
        }
        return LSM_RET_DEFAULT(setprocattr);
}

/**
 * security_netlink_send() - Save info and check if netlink sending is allowed
 * @sk: sending socket
 * @skb: netlink message
 *
 * Save security information for a netlink message so that permission checking
 * can be performed when the message is processed.  The security information
 * can be saved using the eff_cap field of the netlink_skb_parms structure.
 * Also may be used to provide fine grained control over message transmission.
 *
 * Return: Returns 0 if the information was successfully saved and message is
 *         allowed to be transmitted.
 */
int security_netlink_send(struct sock *sk, struct sk_buff *skb)
{
        return call_int_hook(netlink_send, sk, skb);
}

/**
 * security_ismaclabel() - Check if the named attribute is a MAC label
 * @name: full extended attribute name
 *
 * Check if the extended attribute specified by @name represents a MAC label.
 *
 * Return: Returns 1 if name is a MAC attribute otherwise returns 0.
 */
int security_ismaclabel(const char *name)
{
        return call_int_hook(ismaclabel, name);
}
EXPORT_SYMBOL(security_ismaclabel);

/**
 * security_secid_to_secctx() - Convert a secid to a secctx
 * @secid: secid
 * @secdata: secctx
 * @seclen: secctx length
 *
 * Convert secid to security context.  If @secdata is NULL the length of the
 * result will be returned in @seclen, but no @secdata will be returned.  This
 * does mean that the length could change between calls to check the length and
 * the next call which actually allocates and returns the @secdata.
 *
 * Return: Return 0 on success, error on failure.
 */
int security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen)
{
        return call_int_hook(secid_to_secctx, secid, secdata, seclen);
}
EXPORT_SYMBOL(security_secid_to_secctx);

/**
 * security_lsmprop_to_secctx() - Convert a lsm_prop to a secctx
 * @prop: lsm specific information
 * @secdata: secctx
 * @seclen: secctx length
 *
 * Convert a @prop entry to security context.  If @secdata is NULL the
 * length of the result will be returned in @seclen, but no @secdata
 * will be returned.  This does mean that the length could change between
 * calls to check the length and the next call which actually allocates
 * and returns the @secdata.
 *
 * Return: Return 0 on success, error on failure.
 */
int security_lsmprop_to_secctx(struct lsm_prop *prop, char **secdata,
                               u32 *seclen)
{
        return call_int_hook(lsmprop_to_secctx, prop, secdata, seclen);
}
EXPORT_SYMBOL(security_lsmprop_to_secctx);

/**
 * security_secctx_to_secid() - Convert a secctx to a secid
 * @secdata: secctx
 * @seclen: length of secctx
 * @secid: secid
 *
 * Convert security context to secid.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_secctx_to_secid(const char *secdata, u32 seclen, u32 *secid)
{
        *secid = 0;
        return call_int_hook(secctx_to_secid, secdata, seclen, secid);
}
EXPORT_SYMBOL(security_secctx_to_secid);

/**
 * security_release_secctx() - Free a secctx buffer
 * @secdata: secctx
 * @seclen: length of secctx
 *
 * Release the security context.
 */
void security_release_secctx(char *secdata, u32 seclen)
{
        call_void_hook(release_secctx, secdata, seclen);
}
EXPORT_SYMBOL(security_release_secctx);

/**
 * security_inode_invalidate_secctx() - Invalidate an inode's security label
 * @inode: inode
 *
 * Notify the security module that it must revalidate the security context of
 * an inode.
 */
void security_inode_invalidate_secctx(struct inode *inode)
{
        call_void_hook(inode_invalidate_secctx, inode);
}
EXPORT_SYMBOL(security_inode_invalidate_secctx);

/**
 * security_inode_notifysecctx() - Notify the LSM of an inode's security label
 * @inode: inode
 * @ctx: secctx
 * @ctxlen: length of secctx
 *
 * Notify the security module of what the security context of an inode should
 * be.  Initializes the incore security context managed by the security module
 * for this inode.  Example usage: NFS client invokes this hook to initialize
 * the security context in its incore inode to the value provided by the server
 * for the file when the server returned the file's attributes to the client.
 * Must be called with inode->i_mutex locked.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen)
{
        return call_int_hook(inode_notifysecctx, inode, ctx, ctxlen);
}
EXPORT_SYMBOL(security_inode_notifysecctx);

/**
 * security_inode_setsecctx() - Change the security label of an inode
 * @dentry: inode
 * @ctx: secctx
 * @ctxlen: length of secctx
 *
 * Change the security context of an inode.  Updates the incore security
 * context managed by the security module and invokes the fs code as needed
 * (via __vfs_setxattr_noperm) to update any backing xattrs that represent the
 * context.  Example usage: NFS server invokes this hook to change the security
 * context in its incore inode and on the backing filesystem to a value
 * provided by the client on a SETATTR operation.  Must be called with
 * inode->i_mutex locked.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen)
{
        return call_int_hook(inode_setsecctx, dentry, ctx, ctxlen);
}
EXPORT_SYMBOL(security_inode_setsecctx);

/**
 * security_inode_getsecctx() - Get the security label of an inode
 * @inode: inode
 * @ctx: secctx
 * @ctxlen: length of secctx
 *
 * On success, returns 0 and fills out @ctx and @ctxlen with the security
 * context for the given @inode.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen)
{
        return call_int_hook(inode_getsecctx, inode, ctx, ctxlen);
}
EXPORT_SYMBOL(security_inode_getsecctx);

#ifdef CONFIG_WATCH_QUEUE
/**
 * security_post_notification() - Check if a watch notification can be posted
 * @w_cred: credentials of the task that set the watch
 * @cred: credentials of the task which triggered the watch
 * @n: the notification
 *
 * Check to see if a watch notification can be posted to a particular queue.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_post_notification(const struct cred *w_cred,
                               const struct cred *cred,
                               struct watch_notification *n)
{
        return call_int_hook(post_notification, w_cred, cred, n);
}
#endif /* CONFIG_WATCH_QUEUE */

#ifdef CONFIG_KEY_NOTIFICATIONS
/**
 * security_watch_key() - Check if a task is allowed to watch for key events
 * @key: the key to watch
 *
 * Check to see if a process is allowed to watch for event notifications from
 * a key or keyring.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_watch_key(struct key *key)
{
        return call_int_hook(watch_key, key);
}
#endif /* CONFIG_KEY_NOTIFICATIONS */

#ifdef CONFIG_SECURITY_NETWORK
/**
 * security_unix_stream_connect() - Check if a AF_UNIX stream is allowed
 * @sock: originating sock
 * @other: peer sock
 * @newsk: new sock
 *
 * Check permissions before establishing a Unix domain stream connection
 * between @sock and @other.
 *
 * The @unix_stream_connect and @unix_may_send hooks were necessary because
 * Linux provides an alternative to the conventional file name space for Unix
 * domain sockets.  Whereas binding and connecting to sockets in the file name
 * space is mediated by the typical file permissions (and caught by the mknod
 * and permission hooks in inode_security_ops), binding and connecting to
 * sockets in the abstract name space is completely unmediated.  Sufficient
 * control of Unix domain sockets in the abstract name space isn't possible
 * using only the socket layer hooks, since we need to know the actual target
 * socket, which is not looked up until we are inside the af_unix code.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_unix_stream_connect(struct sock *sock, struct sock *other,
                                 struct sock *newsk)
{
        return call_int_hook(unix_stream_connect, sock, other, newsk);
}
EXPORT_SYMBOL(security_unix_stream_connect);

/**
 * security_unix_may_send() - Check if AF_UNIX socket can send datagrams
 * @sock: originating sock
 * @other: peer sock
 *
 * Check permissions before connecting or sending datagrams from @sock to
 * @other.
 *
 * The @unix_stream_connect and @unix_may_send hooks were necessary because
 * Linux provides an alternative to the conventional file name space for Unix
 * domain sockets.  Whereas binding and connecting to sockets in the file name
 * space is mediated by the typical file permissions (and caught by the mknod
 * and permission hooks in inode_security_ops), binding and connecting to
 * sockets in the abstract name space is completely unmediated.  Sufficient
 * control of Unix domain sockets in the abstract name space isn't possible
 * using only the socket layer hooks, since we need to know the actual target
 * socket, which is not looked up until we are inside the af_unix code.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_unix_may_send(struct socket *sock,  struct socket *other)
{
        return call_int_hook(unix_may_send, sock, other);
}
EXPORT_SYMBOL(security_unix_may_send);

/**
 * security_socket_create() - Check if creating a new socket is allowed
 * @family: protocol family
 * @type: communications type
 * @protocol: requested protocol
 * @kern: set to 1 if a kernel socket is requested
 *
 * Check permissions prior to creating a new socket.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_create(int family, int type, int protocol, int kern)
{
        return call_int_hook(socket_create, family, type, protocol, kern);
}

/**
 * security_socket_post_create() - Initialize a newly created socket
 * @sock: socket
 * @family: protocol family
 * @type: communications type
 * @protocol: requested protocol
 * @kern: set to 1 if a kernel socket is requested
 *
 * This hook allows a module to update or allocate a per-socket security
 * structure. Note that the security field was not added directly to the socket
 * structure, but rather, the socket security information is stored in the
 * associated inode.  Typically, the inode alloc_security hook will allocate
 * and attach security information to SOCK_INODE(sock)->i_security.  This hook
 * may be used to update the SOCK_INODE(sock)->i_security field with additional
 * information that wasn't available when the inode was allocated.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_post_create(struct socket *sock, int family,
                                int type, int protocol, int kern)
{
        return call_int_hook(socket_post_create, sock, family, type,
                             protocol, kern);
}

/**
 * security_socket_socketpair() - Check if creating a socketpair is allowed
 * @socka: first socket
 * @sockb: second socket
 *
 * Check permissions before creating a fresh pair of sockets.
 *
 * Return: Returns 0 if permission is granted and the connection was
 *         established.
 */
int security_socket_socketpair(struct socket *socka, struct socket *sockb)
{
        return call_int_hook(socket_socketpair, socka, sockb);
}
EXPORT_SYMBOL(security_socket_socketpair);

/**
 * security_socket_bind() - Check if a socket bind operation is allowed
 * @sock: socket
 * @address: requested bind address
 * @addrlen: length of address
 *
 * Check permission before socket protocol layer bind operation is performed
 * and the socket @sock is bound to the address specified in the @address
 * parameter.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_bind(struct socket *sock,
                         struct sockaddr *address, int addrlen)
{
        return call_int_hook(socket_bind, sock, address, addrlen);
}

/**
 * security_socket_connect() - Check if a socket connect operation is allowed
 * @sock: socket
 * @address: address of remote connection point
 * @addrlen: length of address
 *
 * Check permission before socket protocol layer connect operation attempts to
 * connect socket @sock to a remote address, @address.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_connect(struct socket *sock,
                            struct sockaddr *address, int addrlen)
{
        return call_int_hook(socket_connect, sock, address, addrlen);
}

/**
 * security_socket_listen() - Check if a socket is allowed to listen
 * @sock: socket
 * @backlog: connection queue size
 *
 * Check permission before socket protocol layer listen operation.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_listen(struct socket *sock, int backlog)
{
        return call_int_hook(socket_listen, sock, backlog);
}

/**
 * security_socket_accept() - Check if a socket is allowed to accept connections
 * @sock: listening socket
 * @newsock: newly creation connection socket
 *
 * Check permission before accepting a new connection.  Note that the new
 * socket, @newsock, has been created and some information copied to it, but
 * the accept operation has not actually been performed.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_accept(struct socket *sock, struct socket *newsock)
{
        return call_int_hook(socket_accept, sock, newsock);
}

/**
 * security_socket_sendmsg() - Check if sending a message is allowed
 * @sock: sending socket
 * @msg: message to send
 * @size: size of message
 *
 * Check permission before transmitting a message to another socket.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_sendmsg(struct socket *sock, struct msghdr *msg, int size)
{
        return call_int_hook(socket_sendmsg, sock, msg, size);
}

/**
 * security_socket_recvmsg() - Check if receiving a message is allowed
 * @sock: receiving socket
 * @msg: message to receive
 * @size: size of message
 * @flags: operational flags
 *
 * Check permission before receiving a message from a socket.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_recvmsg(struct socket *sock, struct msghdr *msg,
                            int size, int flags)
{
        return call_int_hook(socket_recvmsg, sock, msg, size, flags);
}

/**
 * security_socket_getsockname() - Check if reading the socket addr is allowed
 * @sock: socket
 *
 * Check permission before reading the local address (name) of the socket
 * object.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_getsockname(struct socket *sock)
{
        return call_int_hook(socket_getsockname, sock);
}

/**
 * security_socket_getpeername() - Check if reading the peer's addr is allowed
 * @sock: socket
 *
 * Check permission before the remote address (name) of a socket object.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_getpeername(struct socket *sock)
{
        return call_int_hook(socket_getpeername, sock);
}

/**
 * security_socket_getsockopt() - Check if reading a socket option is allowed
 * @sock: socket
 * @level: option's protocol level
 * @optname: option name
 *
 * Check permissions before retrieving the options associated with socket
 * @sock.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_getsockopt(struct socket *sock, int level, int optname)
{
        return call_int_hook(socket_getsockopt, sock, level, optname);
}

/**
 * security_socket_setsockopt() - Check if setting a socket option is allowed
 * @sock: socket
 * @level: option's protocol level
 * @optname: option name
 *
 * Check permissions before setting the options associated with socket @sock.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_setsockopt(struct socket *sock, int level, int optname)
{
        return call_int_hook(socket_setsockopt, sock, level, optname);
}

/**
 * security_socket_shutdown() - Checks if shutting down the socket is allowed
 * @sock: socket
 * @how: flag indicating how sends and receives are handled
 *
 * Checks permission before all or part of a connection on the socket @sock is
 * shut down.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_socket_shutdown(struct socket *sock, int how)
{
        return call_int_hook(socket_shutdown, sock, how);
}

/**
 * security_sock_rcv_skb() - Check if an incoming network packet is allowed
 * @sk: destination sock
 * @skb: incoming packet
 *
 * Check permissions on incoming network packets.  This hook is distinct from
 * Netfilter's IP input hooks since it is the first time that the incoming
 * sk_buff @skb has been associated with a particular socket, @sk.  Must not
 * sleep inside this hook because some callers hold spinlocks.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sock_rcv_skb(struct sock *sk, struct sk_buff *skb)
{
        return call_int_hook(socket_sock_rcv_skb, sk, skb);
}
EXPORT_SYMBOL(security_sock_rcv_skb);

/**
 * security_socket_getpeersec_stream() - Get the remote peer label
 * @sock: socket
 * @optval: destination buffer
 * @optlen: size of peer label copied into the buffer
 * @len: maximum size of the destination buffer
 *
 * This hook allows the security module to provide peer socket security state
 * for unix or connected tcp sockets to userspace via getsockopt SO_GETPEERSEC.
 * For tcp sockets this can be meaningful if the socket is associated with an
 * ipsec SA.
 *
 * Return: Returns 0 if all is well, otherwise, typical getsockopt return
 *         values.
 */
int security_socket_getpeersec_stream(struct socket *sock, sockptr_t optval,
                                      sockptr_t optlen, unsigned int len)
{
        return call_int_hook(socket_getpeersec_stream, sock, optval, optlen,
                             len);
}

/**
 * security_socket_getpeersec_dgram() - Get the remote peer label
 * @sock: socket
 * @skb: datagram packet
 * @secid: remote peer label secid
 *
 * This hook allows the security module to provide peer socket security state
 * for udp sockets on a per-packet basis to userspace via getsockopt
 * SO_GETPEERSEC. The application must first have indicated the IP_PASSSEC
 * option via getsockopt. It can then retrieve the security state returned by
 * this hook for a packet via the SCM_SECURITY ancillary message type.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_socket_getpeersec_dgram(struct socket *sock,
                                     struct sk_buff *skb, u32 *secid)
{
        return call_int_hook(socket_getpeersec_dgram, sock, skb, secid);
}
EXPORT_SYMBOL(security_socket_getpeersec_dgram);

/**
 * lsm_sock_alloc - allocate a composite sock blob
 * @sock: the sock that needs a blob
 * @gfp: allocation mode
 *
 * Allocate the sock blob for all the modules
 *
 * Returns 0, or -ENOMEM if memory can't be allocated.
 */
static int lsm_sock_alloc(struct sock *sock, gfp_t gfp)
{
        return lsm_blob_alloc(&sock->sk_security, blob_sizes.lbs_sock, gfp);
}

/**
 * security_sk_alloc() - Allocate and initialize a sock's LSM blob
 * @sk: sock
 * @family: protocol family
 * @priority: gfp flags
 *
 * Allocate and attach a security structure to the sk->sk_security field, which
 * is used to copy security attributes between local stream sockets.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_sk_alloc(struct sock *sk, int family, gfp_t priority)
{
        int rc = lsm_sock_alloc(sk, priority);

        if (unlikely(rc))
                return rc;
        rc = call_int_hook(sk_alloc_security, sk, family, priority);
        if (unlikely(rc))
                security_sk_free(sk);
        return rc;
}

/**
 * security_sk_free() - Free the sock's LSM blob
 * @sk: sock
 *
 * Deallocate security structure.
 */
void security_sk_free(struct sock *sk)
{
        call_void_hook(sk_free_security, sk);
        kfree(sk->sk_security);
        sk->sk_security = NULL;
}

/**
 * security_sk_clone() - Clone a sock's LSM state
 * @sk: original sock
 * @newsk: target sock
 *
 * Clone/copy security structure.
 */
void security_sk_clone(const struct sock *sk, struct sock *newsk)
{
        call_void_hook(sk_clone_security, sk, newsk);
}
EXPORT_SYMBOL(security_sk_clone);

/**
 * security_sk_classify_flow() - Set a flow's secid based on socket
 * @sk: original socket
 * @flic: target flow
 *
 * Set the target flow's secid to socket's secid.
 */
void security_sk_classify_flow(const struct sock *sk, struct flowi_common *flic)
{
        call_void_hook(sk_getsecid, sk, &flic->flowic_secid);
}
EXPORT_SYMBOL(security_sk_classify_flow);

/**
 * security_req_classify_flow() - Set a flow's secid based on request_sock
 * @req: request_sock
 * @flic: target flow
 *
 * Sets @flic's secid to @req's secid.
 */
void security_req_classify_flow(const struct request_sock *req,
                                struct flowi_common *flic)
{
        call_void_hook(req_classify_flow, req, flic);
}
EXPORT_SYMBOL(security_req_classify_flow);

/**
 * security_sock_graft() - Reconcile LSM state when grafting a sock on a socket
 * @sk: sock being grafted
 * @parent: target parent socket
 *
 * Sets @parent's inode secid to @sk's secid and update @sk with any necessary
 * LSM state from @parent.
 */
void security_sock_graft(struct sock *sk, struct socket *parent)
{
        call_void_hook(sock_graft, sk, parent);
}
EXPORT_SYMBOL(security_sock_graft);

/**
 * security_inet_conn_request() - Set request_sock state using incoming connect
 * @sk: parent listening sock
 * @skb: incoming connection
 * @req: new request_sock
 *
 * Initialize the @req LSM state based on @sk and the incoming connect in @skb.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_inet_conn_request(const struct sock *sk,
                               struct sk_buff *skb, struct request_sock *req)
{
        return call_int_hook(inet_conn_request, sk, skb, req);
}
EXPORT_SYMBOL(security_inet_conn_request);

/**
 * security_inet_csk_clone() - Set new sock LSM state based on request_sock
 * @newsk: new sock
 * @req: connection request_sock
 *
 * Set that LSM state of @sock using the LSM state from @req.
 */
void security_inet_csk_clone(struct sock *newsk,
                             const struct request_sock *req)
{
        call_void_hook(inet_csk_clone, newsk, req);
}

/**
 * security_inet_conn_established() - Update sock's LSM state with connection
 * @sk: sock
 * @skb: connection packet
 *
 * Update @sock's LSM state to represent a new connection from @skb.
 */
void security_inet_conn_established(struct sock *sk,
                                    struct sk_buff *skb)
{
        call_void_hook(inet_conn_established, sk, skb);
}
EXPORT_SYMBOL(security_inet_conn_established);

/**
 * security_secmark_relabel_packet() - Check if setting a secmark is allowed
 * @secid: new secmark value
 *
 * Check if the process should be allowed to relabel packets to @secid.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_secmark_relabel_packet(u32 secid)
{
        return call_int_hook(secmark_relabel_packet, secid);
}
EXPORT_SYMBOL(security_secmark_relabel_packet);

/**
 * security_secmark_refcount_inc() - Increment the secmark labeling rule count
 *
 * Tells the LSM to increment the number of secmark labeling rules loaded.
 */
void security_secmark_refcount_inc(void)
{
        call_void_hook(secmark_refcount_inc);
}
EXPORT_SYMBOL(security_secmark_refcount_inc);

/**
 * security_secmark_refcount_dec() - Decrement the secmark labeling rule count
 *
 * Tells the LSM to decrement the number of secmark labeling rules loaded.
 */
void security_secmark_refcount_dec(void)
{
        call_void_hook(secmark_refcount_dec);
}
EXPORT_SYMBOL(security_secmark_refcount_dec);

/**
 * security_tun_dev_alloc_security() - Allocate a LSM blob for a TUN device
 * @security: pointer to the LSM blob
 *
 * This hook allows a module to allocate a security structure for a TUN        device,
 * returning the pointer in @security.
 *
 * Return: Returns a zero on success, negative values on failure.
 */
int security_tun_dev_alloc_security(void **security)
{
        int rc;

        rc = lsm_blob_alloc(security, blob_sizes.lbs_tun_dev, GFP_KERNEL);
        if (rc)
                return rc;

        rc = call_int_hook(tun_dev_alloc_security, *security);
        if (rc) {
                kfree(*security);
                *security = NULL;
        }
        return rc;
}
EXPORT_SYMBOL(security_tun_dev_alloc_security);

/**
 * security_tun_dev_free_security() - Free a TUN device LSM blob
 * @security: LSM blob
 *
 * This hook allows a module to free the security structure for a TUN device.
 */
void security_tun_dev_free_security(void *security)
{
        kfree(security);
}
EXPORT_SYMBOL(security_tun_dev_free_security);

/**
 * security_tun_dev_create() - Check if creating a TUN device is allowed
 *
 * Check permissions prior to creating a new TUN device.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_tun_dev_create(void)
{
        return call_int_hook(tun_dev_create);
}
EXPORT_SYMBOL(security_tun_dev_create);

/**
 * security_tun_dev_attach_queue() - Check if attaching a TUN queue is allowed
 * @security: TUN device LSM blob
 *
 * Check permissions prior to attaching to a TUN device queue.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_tun_dev_attach_queue(void *security)
{
        return call_int_hook(tun_dev_attach_queue, security);
}
EXPORT_SYMBOL(security_tun_dev_attach_queue);

/**
 * security_tun_dev_attach() - Update TUN device LSM state on attach
 * @sk: associated sock
 * @security: TUN device LSM blob
 *
 * This hook can be used by the module to update any security state associated
 * with the TUN device's sock structure.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_tun_dev_attach(struct sock *sk, void *security)
{
        return call_int_hook(tun_dev_attach, sk, security);
}
EXPORT_SYMBOL(security_tun_dev_attach);

/**
 * security_tun_dev_open() - Update TUN device LSM state on open
 * @security: TUN device LSM blob
 *
 * This hook can be used by the module to update any security state associated
 * with the TUN device's security structure.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_tun_dev_open(void *security)
{
        return call_int_hook(tun_dev_open, security);
}
EXPORT_SYMBOL(security_tun_dev_open);

/**
 * security_sctp_assoc_request() - Update the LSM on a SCTP association req
 * @asoc: SCTP association
 * @skb: packet requesting the association
 *
 * Passes the @asoc and @chunk->skb of the association INIT packet to the LSM.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_sctp_assoc_request(struct sctp_association *asoc,
                                struct sk_buff *skb)
{
        return call_int_hook(sctp_assoc_request, asoc, skb);
}
EXPORT_SYMBOL(security_sctp_assoc_request);

/**
 * security_sctp_bind_connect() - Validate a list of addrs for a SCTP option
 * @sk: socket
 * @optname: SCTP option to validate
 * @address: list of IP addresses to validate
 * @addrlen: length of the address list
 *
 * Validiate permissions required for each address associated with sock        @sk.
 * Depending on @optname, the addresses will be treated as either a connect or
 * bind service. The @addrlen is calculated on each IPv4 and IPv6 address using
 * sizeof(struct sockaddr_in) or sizeof(struct sockaddr_in6).
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_sctp_bind_connect(struct sock *sk, int optname,
                               struct sockaddr *address, int addrlen)
{
        return call_int_hook(sctp_bind_connect, sk, optname, address, addrlen);
}
EXPORT_SYMBOL(security_sctp_bind_connect);

/**
 * security_sctp_sk_clone() - Clone a SCTP sock's LSM state
 * @asoc: SCTP association
 * @sk: original sock
 * @newsk: target sock
 *
 * Called whenever a new socket is created by accept(2) (i.e. a TCP style
 * socket) or when a socket is 'peeled off' e.g userspace calls
 * sctp_peeloff(3).
 */
void security_sctp_sk_clone(struct sctp_association *asoc, struct sock *sk,
                            struct sock *newsk)
{
        call_void_hook(sctp_sk_clone, asoc, sk, newsk);
}
EXPORT_SYMBOL(security_sctp_sk_clone);

/**
 * security_sctp_assoc_established() - Update LSM state when assoc established
 * @asoc: SCTP association
 * @skb: packet establishing the association
 *
 * Passes the @asoc and @chunk->skb of the association COOKIE_ACK packet to the
 * security module.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_sctp_assoc_established(struct sctp_association *asoc,
                                    struct sk_buff *skb)
{
        return call_int_hook(sctp_assoc_established, asoc, skb);
}
EXPORT_SYMBOL(security_sctp_assoc_established);

/**
 * security_mptcp_add_subflow() - Inherit the LSM label from the MPTCP socket
 * @sk: the owning MPTCP socket
 * @ssk: the new subflow
 *
 * Update the labeling for the given MPTCP subflow, to match the one of the
 * owning MPTCP socket. This hook has to be called after the socket creation and
 * initialization via the security_socket_create() and
 * security_socket_post_create() LSM hooks.
 *
 * Return: Returns 0 on success or a negative error code on failure.
 */
int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk)
{
        return call_int_hook(mptcp_add_subflow, sk, ssk);
}

#endif        /* CONFIG_SECURITY_NETWORK */

#ifdef CONFIG_SECURITY_INFINIBAND
/**
 * security_ib_pkey_access() - Check if access to an IB pkey is allowed
 * @sec: LSM blob
 * @subnet_prefix: subnet prefix of the port
 * @pkey: IB pkey
 *
 * Check permission to access a pkey when modifying a QP.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_ib_pkey_access(void *sec, u64 subnet_prefix, u16 pkey)
{
        return call_int_hook(ib_pkey_access, sec, subnet_prefix, pkey);
}
EXPORT_SYMBOL(security_ib_pkey_access);

/**
 * security_ib_endport_manage_subnet() - Check if SMPs traffic is allowed
 * @sec: LSM blob
 * @dev_name: IB device name
 * @port_num: port number
 *
 * Check permissions to send and receive SMPs on a end port.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_ib_endport_manage_subnet(void *sec,
                                      const char *dev_name, u8 port_num)
{
        return call_int_hook(ib_endport_manage_subnet, sec, dev_name, port_num);
}
EXPORT_SYMBOL(security_ib_endport_manage_subnet);

/**
 * security_ib_alloc_security() - Allocate an Infiniband LSM blob
 * @sec: LSM blob
 *
 * Allocate a security structure for Infiniband objects.
 *
 * Return: Returns 0 on success, non-zero on failure.
 */
int security_ib_alloc_security(void **sec)
{
        int rc;

        rc = lsm_blob_alloc(sec, blob_sizes.lbs_ib, GFP_KERNEL);
        if (rc)
                return rc;

        rc = call_int_hook(ib_alloc_security, *sec);
        if (rc) {
                kfree(*sec);
                *sec = NULL;
        }
        return rc;
}
EXPORT_SYMBOL(security_ib_alloc_security);

/**
 * security_ib_free_security() - Free an Infiniband LSM blob
 * @sec: LSM blob
 *
 * Deallocate an Infiniband security structure.
 */
void security_ib_free_security(void *sec)
{
        kfree(sec);
}
EXPORT_SYMBOL(security_ib_free_security);
#endif        /* CONFIG_SECURITY_INFINIBAND */

#ifdef CONFIG_SECURITY_NETWORK_XFRM
/**
 * security_xfrm_policy_alloc() - Allocate a xfrm policy LSM blob
 * @ctxp: xfrm security context being added to the SPD
 * @sec_ctx: security label provided by userspace
 * @gfp: gfp flags
 *
 * Allocate a security structure to the xp->security field; the security field
 * is initialized to NULL when the xfrm_policy is allocated.
 *
 * Return:  Return 0 if operation was successful.
 */
int security_xfrm_policy_alloc(struct xfrm_sec_ctx **ctxp,
                               struct xfrm_user_sec_ctx *sec_ctx,
                               gfp_t gfp)
{
        return call_int_hook(xfrm_policy_alloc_security, ctxp, sec_ctx, gfp);
}
EXPORT_SYMBOL(security_xfrm_policy_alloc);

/**
 * security_xfrm_policy_clone() - Clone xfrm policy LSM state
 * @old_ctx: xfrm security context
 * @new_ctxp: target xfrm security context
 *
 * Allocate a security structure in new_ctxp that contains the information from
 * the old_ctx structure.
 *
 * Return: Return 0 if operation was successful.
 */
int security_xfrm_policy_clone(struct xfrm_sec_ctx *old_ctx,
                               struct xfrm_sec_ctx **new_ctxp)
{
        return call_int_hook(xfrm_policy_clone_security, old_ctx, new_ctxp);
}

/**
 * security_xfrm_policy_free() - Free a xfrm security context
 * @ctx: xfrm security context
 *
 * Free LSM resources associated with @ctx.
 */
void security_xfrm_policy_free(struct xfrm_sec_ctx *ctx)
{
        call_void_hook(xfrm_policy_free_security, ctx);
}
EXPORT_SYMBOL(security_xfrm_policy_free);

/**
 * security_xfrm_policy_delete() - Check if deleting a xfrm policy is allowed
 * @ctx: xfrm security context
 *
 * Authorize deletion of a SPD entry.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_xfrm_policy_delete(struct xfrm_sec_ctx *ctx)
{
        return call_int_hook(xfrm_policy_delete_security, ctx);
}

/**
 * security_xfrm_state_alloc() - Allocate a xfrm state LSM blob
 * @x: xfrm state being added to the SAD
 * @sec_ctx: security label provided by userspace
 *
 * Allocate a security structure to the @x->security field; the security field
 * is initialized to NULL when the xfrm_state is allocated. Set the context to
 * correspond to @sec_ctx.
 *
 * Return: Return 0 if operation was successful.
 */
int security_xfrm_state_alloc(struct xfrm_state *x,
                              struct xfrm_user_sec_ctx *sec_ctx)
{
        return call_int_hook(xfrm_state_alloc, x, sec_ctx);
}
EXPORT_SYMBOL(security_xfrm_state_alloc);

/**
 * security_xfrm_state_alloc_acquire() - Allocate a xfrm state LSM blob
 * @x: xfrm state being added to the SAD
 * @polsec: associated policy's security context
 * @secid: secid from the flow
 *
 * Allocate a security structure to the x->security field; the security field
 * is initialized to NULL when the xfrm_state is allocated.  Set the context to
 * correspond to secid.
 *
 * Return: Returns 0 if operation was successful.
 */
int security_xfrm_state_alloc_acquire(struct xfrm_state *x,
                                      struct xfrm_sec_ctx *polsec, u32 secid)
{
        return call_int_hook(xfrm_state_alloc_acquire, x, polsec, secid);
}

/**
 * security_xfrm_state_delete() - Check if deleting a xfrm state is allowed
 * @x: xfrm state
 *
 * Authorize deletion of x->security.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_xfrm_state_delete(struct xfrm_state *x)
{
        return call_int_hook(xfrm_state_delete_security, x);
}
EXPORT_SYMBOL(security_xfrm_state_delete);

/**
 * security_xfrm_state_free() - Free a xfrm state
 * @x: xfrm state
 *
 * Deallocate x->security.
 */
void security_xfrm_state_free(struct xfrm_state *x)
{
        call_void_hook(xfrm_state_free_security, x);
}

/**
 * security_xfrm_policy_lookup() - Check if using a xfrm policy is allowed
 * @ctx: target xfrm security context
 * @fl_secid: flow secid used to authorize access
 *
 * Check permission when a flow selects a xfrm_policy for processing XFRMs on a
 * packet.  The hook is called when selecting either a per-socket policy or a
 * generic xfrm policy.
 *
 * Return: Return 0 if permission is granted, -ESRCH otherwise, or -errno on
 *         other errors.
 */
int security_xfrm_policy_lookup(struct xfrm_sec_ctx *ctx, u32 fl_secid)
{
        return call_int_hook(xfrm_policy_lookup, ctx, fl_secid);
}

/**
 * security_xfrm_state_pol_flow_match() - Check for a xfrm match
 * @x: xfrm state to match
 * @xp: xfrm policy to check for a match
 * @flic: flow to check for a match.
 *
 * Check @xp and @flic for a match with @x.
 *
 * Return: Returns 1 if there is a match.
 */
int security_xfrm_state_pol_flow_match(struct xfrm_state *x,
                                       struct xfrm_policy *xp,
                                       const struct flowi_common *flic)
{
        struct lsm_static_call *scall;
        int rc = LSM_RET_DEFAULT(xfrm_state_pol_flow_match);

        /*
         * Since this function is expected to return 0 or 1, the judgment
         * becomes difficult if multiple LSMs supply this call. Fortunately,
         * we can use the first LSM's judgment because currently only SELinux
         * supplies this call.
         *
         * For speed optimization, we explicitly break the loop rather than
         * using the macro
         */
        lsm_for_each_hook(scall, xfrm_state_pol_flow_match) {
                rc = scall->hl->hook.xfrm_state_pol_flow_match(x, xp, flic);
                break;
        }
        return rc;
}

/**
 * security_xfrm_decode_session() - Determine the xfrm secid for a packet
 * @skb: xfrm packet
 * @secid: secid
 *
 * Decode the packet in @skb and return the security label in @secid.
 *
 * Return: Return 0 if all xfrms used have the same secid.
 */
int security_xfrm_decode_session(struct sk_buff *skb, u32 *secid)
{
        return call_int_hook(xfrm_decode_session, skb, secid, 1);
}

void security_skb_classify_flow(struct sk_buff *skb, struct flowi_common *flic)
{
        int rc = call_int_hook(xfrm_decode_session, skb, &flic->flowic_secid,
                               0);

        BUG_ON(rc);
}
EXPORT_SYMBOL(security_skb_classify_flow);
#endif        /* CONFIG_SECURITY_NETWORK_XFRM */

#ifdef CONFIG_KEYS
/**
 * security_key_alloc() - Allocate and initialize a kernel key LSM blob
 * @key: key
 * @cred: credentials
 * @flags: allocation flags
 *
 * Permit allocation of a key and assign security data. Note that key does not
 * have a serial number assigned at this point.
 *
 * Return: Return 0 if permission is granted, -ve error otherwise.
 */
int security_key_alloc(struct key *key, const struct cred *cred,
                       unsigned long flags)
{
        int rc = lsm_key_alloc(key);

        if (unlikely(rc))
                return rc;
        rc = call_int_hook(key_alloc, key, cred, flags);
        if (unlikely(rc))
                security_key_free(key);
        return rc;
}

/**
 * security_key_free() - Free a kernel key LSM blob
 * @key: key
 *
 * Notification of destruction; free security data.
 */
void security_key_free(struct key *key)
{
        kfree(key->security);
        key->security = NULL;
}

/**
 * security_key_permission() - Check if a kernel key operation is allowed
 * @key_ref: key reference
 * @cred: credentials of actor requesting access
 * @need_perm: requested permissions
 *
 * See whether a specific operational right is granted to a process on a key.
 *
 * Return: Return 0 if permission is granted, -ve error otherwise.
 */
int security_key_permission(key_ref_t key_ref, const struct cred *cred,
                            enum key_need_perm need_perm)
{
        return call_int_hook(key_permission, key_ref, cred, need_perm);
}

/**
 * security_key_getsecurity() - Get the key's security label
 * @key: key
 * @buffer: security label buffer
 *
 * Get a textual representation of the security context attached to a key for
 * the purposes of honouring KEYCTL_GETSECURITY.  This function allocates the
 * storage for the NUL-terminated string and the caller should free it.
 *
 * Return: Returns the length of @buffer (including terminating NUL) or -ve if
 *         an error occurs.  May also return 0 (and a NULL buffer pointer) if
 *         there is no security label assigned to the key.
 */
int security_key_getsecurity(struct key *key, char **buffer)
{
        *buffer = NULL;
        return call_int_hook(key_getsecurity, key, buffer);
}

/**
 * security_key_post_create_or_update() - Notification of key create or update
 * @keyring: keyring to which the key is linked to
 * @key: created or updated key
 * @payload: data used to instantiate or update the key
 * @payload_len: length of payload
 * @flags: key flags
 * @create: flag indicating whether the key was created or updated
 *
 * Notify the caller of a key creation or update.
 */
void security_key_post_create_or_update(struct key *keyring, struct key *key,
                                        const void *payload, size_t payload_len,
                                        unsigned long flags, bool create)
{
        call_void_hook(key_post_create_or_update, keyring, key, payload,
                       payload_len, flags, create);
}
#endif        /* CONFIG_KEYS */

#ifdef CONFIG_AUDIT
/**
 * security_audit_rule_init() - Allocate and init an LSM audit rule struct
 * @field: audit action
 * @op: rule operator
 * @rulestr: rule context
 * @lsmrule: receive buffer for audit rule struct
 * @gfp: GFP flag used for kmalloc
 *
 * Allocate and initialize an LSM audit rule structure.
 *
 * Return: Return 0 if @lsmrule has been successfully set, -EINVAL in case of
 *         an invalid rule.
 */
int security_audit_rule_init(u32 field, u32 op, char *rulestr, void **lsmrule,
                             gfp_t gfp)
{
        return call_int_hook(audit_rule_init, field, op, rulestr, lsmrule, gfp);
}

/**
 * security_audit_rule_known() - Check if an audit rule contains LSM fields
 * @krule: audit rule
 *
 * Specifies whether given @krule contains any fields related to the current
 * LSM.
 *
 * Return: Returns 1 in case of relation found, 0 otherwise.
 */
int security_audit_rule_known(struct audit_krule *krule)
{
        return call_int_hook(audit_rule_known, krule);
}

/**
 * security_audit_rule_free() - Free an LSM audit rule struct
 * @lsmrule: audit rule struct
 *
 * Deallocate the LSM audit rule structure previously allocated by
 * audit_rule_init().
 */
void security_audit_rule_free(void *lsmrule)
{
        call_void_hook(audit_rule_free, lsmrule);
}

/**
 * security_audit_rule_match() - Check if a label matches an audit rule
 * @prop: security label
 * @field: LSM audit field
 * @op: matching operator
 * @lsmrule: audit rule
 *
 * Determine if given @secid matches a rule previously approved by
 * security_audit_rule_known().
 *
 * Return: Returns 1 if secid matches the rule, 0 if it does not, -ERRNO on
 *         failure.
 */
int security_audit_rule_match(struct lsm_prop *prop, u32 field, u32 op,
                              void *lsmrule)
{
        return call_int_hook(audit_rule_match, prop, field, op, lsmrule);
}
#endif /* CONFIG_AUDIT */

#ifdef CONFIG_BPF_SYSCALL
/**
 * security_bpf() - Check if the bpf syscall operation is allowed
 * @cmd: command
 * @attr: bpf attribute
 * @size: size
 *
 * Do a initial check for all bpf syscalls after the attribute is copied into
 * the kernel. The actual security module can implement their own rules to
 * check the specific cmd they need.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_bpf(int cmd, union bpf_attr *attr, unsigned int size)
{
        return call_int_hook(bpf, cmd, attr, size);
}

/**
 * security_bpf_map() - Check if access to a bpf map is allowed
 * @map: bpf map
 * @fmode: mode
 *
 * Do a check when the kernel generates and returns a file descriptor for eBPF
 * maps.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_bpf_map(struct bpf_map *map, fmode_t fmode)
{
        return call_int_hook(bpf_map, map, fmode);
}

/**
 * security_bpf_prog() - Check if access to a bpf program is allowed
 * @prog: bpf program
 *
 * Do a check when the kernel generates and returns a file descriptor for eBPF
 * programs.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_bpf_prog(struct bpf_prog *prog)
{
        return call_int_hook(bpf_prog, prog);
}

/**
 * security_bpf_map_create() - Check if BPF map creation is allowed
 * @map: BPF map object
 * @attr: BPF syscall attributes used to create BPF map
 * @token: BPF token used to grant user access
 *
 * Do a check when the kernel creates a new BPF map. This is also the
 * point where LSM blob is allocated for LSMs that need them.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
                            struct bpf_token *token)
{
        return call_int_hook(bpf_map_create, map, attr, token);
}

/**
 * security_bpf_prog_load() - Check if loading of BPF program is allowed
 * @prog: BPF program object
 * @attr: BPF syscall attributes used to create BPF program
 * @token: BPF token used to grant user access to BPF subsystem
 *
 * Perform an access control check when the kernel loads a BPF program and
 * allocates associated BPF program object. This hook is also responsible for
 * allocating any required LSM state for the BPF program.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
                           struct bpf_token *token)
{
        return call_int_hook(bpf_prog_load, prog, attr, token);
}

/**
 * security_bpf_token_create() - Check if creating of BPF token is allowed
 * @token: BPF token object
 * @attr: BPF syscall attributes used to create BPF token
 * @path: path pointing to BPF FS mount point from which BPF token is created
 *
 * Do a check when the kernel instantiates a new BPF token object from BPF FS
 * instance. This is also the point where LSM blob can be allocated for LSMs.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_bpf_token_create(struct bpf_token *token, union bpf_attr *attr,
                              const struct path *path)
{
        return call_int_hook(bpf_token_create, token, attr, path);
}

/**
 * security_bpf_token_cmd() - Check if BPF token is allowed to delegate
 * requested BPF syscall command
 * @token: BPF token object
 * @cmd: BPF syscall command requested to be delegated by BPF token
 *
 * Do a check when the kernel decides whether provided BPF token should allow
 * delegation of requested BPF syscall command.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cmd)
{
        return call_int_hook(bpf_token_cmd, token, cmd);
}

/**
 * security_bpf_token_capable() - Check if BPF token is allowed to delegate
 * requested BPF-related capability
 * @token: BPF token object
 * @cap: capabilities requested to be delegated by BPF token
 *
 * Do a check when the kernel decides whether provided BPF token should allow
 * delegation of requested BPF-related capabilities.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_bpf_token_capable(const struct bpf_token *token, int cap)
{
        return call_int_hook(bpf_token_capable, token, cap);
}

/**
 * security_bpf_map_free() - Free a bpf map's LSM blob
 * @map: bpf map
 *
 * Clean up the security information stored inside bpf map.
 */
void security_bpf_map_free(struct bpf_map *map)
{
        call_void_hook(bpf_map_free, map);
}

/**
 * security_bpf_prog_free() - Free a BPF program's LSM blob
 * @prog: BPF program struct
 *
 * Clean up the security information stored inside BPF program.
 */
void security_bpf_prog_free(struct bpf_prog *prog)
{
        call_void_hook(bpf_prog_free, prog);
}

/**
 * security_bpf_token_free() - Free a BPF token's LSM blob
 * @token: BPF token struct
 *
 * Clean up the security information stored inside BPF token.
 */
void security_bpf_token_free(struct bpf_token *token)
{
        call_void_hook(bpf_token_free, token);
}
#endif /* CONFIG_BPF_SYSCALL */

/**
 * security_locked_down() - Check if a kernel feature is allowed
 * @what: requested kernel feature
 *
 * Determine whether a kernel feature that potentially enables arbitrary code
 * execution in kernel space should be permitted.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_locked_down(enum lockdown_reason what)
{
        return call_int_hook(locked_down, what);
}
EXPORT_SYMBOL(security_locked_down);

/**
 * security_bdev_alloc() - Allocate a block device LSM blob
 * @bdev: block device
 *
 * Allocate and attach a security structure to @bdev->bd_security.  The
 * security field is initialized to NULL when the bdev structure is
 * allocated.
 *
 * Return: Return 0 if operation was successful.
 */
int security_bdev_alloc(struct block_device *bdev)
{
        int rc = 0;

        rc = lsm_bdev_alloc(bdev);
        if (unlikely(rc))
                return rc;

        rc = call_int_hook(bdev_alloc_security, bdev);
        if (unlikely(rc))
                security_bdev_free(bdev);

        return rc;
}
EXPORT_SYMBOL(security_bdev_alloc);

/**
 * security_bdev_free() - Free a block device's LSM blob
 * @bdev: block device
 *
 * Deallocate the bdev security structure and set @bdev->bd_security to NULL.
 */
void security_bdev_free(struct block_device *bdev)
{
        if (!bdev->bd_security)
                return;

        call_void_hook(bdev_free_security, bdev);

        kfree(bdev->bd_security);
        bdev->bd_security = NULL;
}
EXPORT_SYMBOL(security_bdev_free);

/**
 * security_bdev_setintegrity() - Set the device's integrity data
 * @bdev: block device
 * @type: type of integrity, e.g. hash digest, signature, etc
 * @value: the integrity value
 * @size: size of the integrity value
 *
 * Register a verified integrity measurement of a bdev with LSMs.
 * LSMs should free the previously saved data if @value is NULL.
 * Please note that the new hook should be invoked every time the security
 * information is updated to keep these data current. For example, in dm-verity,
 * if the mapping table is reloaded and configured to use a different dm-verity
 * target with a new roothash and signing information, the previously stored
 * data in the LSM blob will become obsolete. It is crucial to re-invoke the
 * hook to refresh these data and ensure they are up to date. This necessity
 * arises from the design of device-mapper, where a device-mapper device is
 * first created, and then targets are subsequently loaded into it. These
 * targets can be modified multiple times during the device's lifetime.
 * Therefore, while the LSM blob is allocated during the creation of the block
 * device, its actual contents are not initialized at this stage and can change
 * substantially over time. This includes alterations from data that the LSMs
 * 'trusts' to those they do not, making it essential to handle these changes
 * correctly. Failure to address this dynamic aspect could potentially allow
 * for bypassing LSM checks.
 *
 * Return: Returns 0 on success, negative values on failure.
 */
int security_bdev_setintegrity(struct block_device *bdev,
                               enum lsm_integrity_type type, const void *value,
                               size_t size)
{
        return call_int_hook(bdev_setintegrity, bdev, type, value, size);
}
EXPORT_SYMBOL(security_bdev_setintegrity);

#ifdef CONFIG_PERF_EVENTS
/**
 * security_perf_event_open() - Check if a perf event open is allowed
 * @attr: perf event attribute
 * @type: type of event
 *
 * Check whether the @type of perf_event_open syscall is allowed.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_perf_event_open(struct perf_event_attr *attr, int type)
{
        return call_int_hook(perf_event_open, attr, type);
}

/**
 * security_perf_event_alloc() - Allocate a perf event LSM blob
 * @event: perf event
 *
 * Allocate and save perf_event security info.
 *
 * Return: Returns 0 on success, error on failure.
 */
int security_perf_event_alloc(struct perf_event *event)
{
        int rc;

        rc = lsm_blob_alloc(&event->security, blob_sizes.lbs_perf_event,
                            GFP_KERNEL);
        if (rc)
                return rc;

        rc = call_int_hook(perf_event_alloc, event);
        if (rc) {
                kfree(event->security);
                event->security = NULL;
        }
        return rc;
}

/**
 * security_perf_event_free() - Free a perf event LSM blob
 * @event: perf event
 *
 * Release (free) perf_event security info.
 */
void security_perf_event_free(struct perf_event *event)
{
        kfree(event->security);
        event->security = NULL;
}

/**
 * security_perf_event_read() - Check if reading a perf event label is allowed
 * @event: perf event
 *
 * Read perf_event security info if allowed.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_perf_event_read(struct perf_event *event)
{
        return call_int_hook(perf_event_read, event);
}

/**
 * security_perf_event_write() - Check if writing a perf event label is allowed
 * @event: perf event
 *
 * Write perf_event security info if allowed.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_perf_event_write(struct perf_event *event)
{
        return call_int_hook(perf_event_write, event);
}
#endif /* CONFIG_PERF_EVENTS */

#ifdef CONFIG_IO_URING
/**
 * security_uring_override_creds() - Check if overriding creds is allowed
 * @new: new credentials
 *
 * Check if the current task, executing an io_uring operation, is allowed to
 * override it's credentials with @new.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_uring_override_creds(const struct cred *new)
{
        return call_int_hook(uring_override_creds, new);
}

/**
 * security_uring_sqpoll() - Check if IORING_SETUP_SQPOLL is allowed
 *
 * Check whether the current task is allowed to spawn a io_uring polling thread
 * (IORING_SETUP_SQPOLL).
 *
 * Return: Returns 0 if permission is granted.
 */
int security_uring_sqpoll(void)
{
        return call_int_hook(uring_sqpoll);
}

/**
 * security_uring_cmd() - Check if a io_uring passthrough command is allowed
 * @ioucmd: command
 *
 * Check whether the file_operations uring_cmd is allowed to run.
 *
 * Return: Returns 0 if permission is granted.
 */
int security_uring_cmd(struct io_uring_cmd *ioucmd)
{
        return call_int_hook(uring_cmd, ioucmd);
}
#endif /* CONFIG_IO_URING */

/**
 * security_initramfs_populated() - Notify LSMs that initramfs has been loaded
 *
 * Tells the LSMs the initramfs has been unpacked into the rootfs.
 */
void security_initramfs_populated(void)
{
        call_void_hook(initramfs_populated);
}


























































































   74 





   75 


   75 
   75 
   75 





   75 
   76 























    1 




    1 




    1 
























   76 
   74 


    2 

    2 



    2 
    2 
    2 
    2 


    2 


    2 


    2 




    1 






























































   76 













   75 




   75 




   76 








   75 
   75 

























   75 










   76 
   76 
   75 


   76 




    1 
    1 

























   20 




   19 
    5 


    4 

    2 

    1 

    1 


    1 


   19 


   15 


   14 





    3 

    1 















   12 
   11 



   10 

    9 







    8 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
// SPDX-License-Identifier: GPL-2.0+
/*
 * Restartable sequences system call
 *
 * Copyright (C) 2015, Google, Inc.,
 * Paul Turner <pjt@google.com> and Andrew Hunter <ahh@google.com>
 * Copyright (C) 2015-2018, EfficiOS Inc.,
 * Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 */

#include <linux/sched.h>
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/rseq.h>
#include <linux/types.h>
#include <asm/ptrace.h>

#define CREATE_TRACE_POINTS
#include <trace/events/rseq.h>

/* The original rseq structure size (including padding) is 32 bytes. */
#define ORIG_RSEQ_SIZE                32

#define RSEQ_CS_NO_RESTART_FLAGS (RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT | \
                                  RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL | \
                                  RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE)

/*
 *
 * Restartable sequences are a lightweight interface that allows
 * user-level code to be executed atomically relative to scheduler
 * preemption and signal delivery. Typically used for implementing
 * per-cpu operations.
 *
 * It allows user-space to perform update operations on per-cpu data
 * without requiring heavy-weight atomic operations.
 *
 * Detailed algorithm of rseq user-space assembly sequences:
 *
 *                     init(rseq_cs)
 *                     cpu = TLS->rseq::cpu_id_start
 *   [1]               TLS->rseq::rseq_cs = rseq_cs
 *   [start_ip]        ----------------------------
 *   [2]               if (cpu != TLS->rseq::cpu_id)
 *                             goto abort_ip;
 *   [3]               <last_instruction_in_cs>
 *   [post_commit_ip]  ----------------------------
 *
 *   The address of jump target abort_ip must be outside the critical
 *   region, i.e.:
 *
 *     [abort_ip] < [start_ip]  || [abort_ip] >= [post_commit_ip]
 *
 *   Steps [2]-[3] (inclusive) need to be a sequence of instructions in
 *   userspace that can handle being interrupted between any of those
 *   instructions, and then resumed to the abort_ip.
 *
 *   1.  Userspace stores the address of the struct rseq_cs assembly
 *       block descriptor into the rseq_cs field of the registered
 *       struct rseq TLS area. This update is performed through a single
 *       store within the inline assembly instruction sequence.
 *       [start_ip]
 *
 *   2.  Userspace tests to check whether the current cpu_id field match
 *       the cpu number loaded before start_ip, branching to abort_ip
 *       in case of a mismatch.
 *
 *       If the sequence is preempted or interrupted by a signal
 *       at or after start_ip and before post_commit_ip, then the kernel
 *       clears TLS->__rseq_abi::rseq_cs, and sets the user-space return
 *       ip to abort_ip before returning to user-space, so the preempted
 *       execution resumes at abort_ip.
 *
 *   3.  Userspace critical section final instruction before
 *       post_commit_ip is the commit. The critical section is
 *       self-terminating.
 *       [post_commit_ip]
 *
 *   4.  <success>
 *
 *   On failure at [2], or if interrupted by preempt or signal delivery
 *   between [1] and [3]:
 *
 *       [abort_ip]
 *   F1. <failure>
 */

static int rseq_update_cpu_node_id(struct task_struct *t)
{
        struct rseq __user *rseq = t->rseq;
        u32 cpu_id = raw_smp_processor_id();
        u32 node_id = cpu_to_node(cpu_id);
        u32 mm_cid = task_mm_cid(t);

        WARN_ON_ONCE((int) mm_cid < 0);
        if (!user_write_access_begin(rseq, t->rseq_len))
                goto efault;
        unsafe_put_user(cpu_id, &rseq->cpu_id_start, efault_end);
        unsafe_put_user(cpu_id, &rseq->cpu_id, efault_end);
        unsafe_put_user(node_id, &rseq->node_id, efault_end);
        unsafe_put_user(mm_cid, &rseq->mm_cid, efault_end);
        /*
         * Additional feature fields added after ORIG_RSEQ_SIZE
         * need to be conditionally updated only if
         * t->rseq_len != ORIG_RSEQ_SIZE.
         */
        user_write_access_end();
        trace_rseq_update(t);
        return 0;

efault_end:
        user_write_access_end();
efault:
        return -EFAULT;
}

static int rseq_reset_rseq_cpu_node_id(struct task_struct *t)
{
        u32 cpu_id_start = 0, cpu_id = RSEQ_CPU_ID_UNINITIALIZED, node_id = 0,
            mm_cid = 0;

        /*
         * Reset cpu_id_start to its initial state (0).
         */
        if (put_user(cpu_id_start, &t->rseq->cpu_id_start))
                return -EFAULT;
        /*
         * Reset cpu_id to RSEQ_CPU_ID_UNINITIALIZED, so any user coming
         * in after unregistration can figure out that rseq needs to be
         * registered again.
         */
        if (put_user(cpu_id, &t->rseq->cpu_id))
                return -EFAULT;
        /*
         * Reset node_id to its initial state (0).
         */
        if (put_user(node_id, &t->rseq->node_id))
                return -EFAULT;
        /*
         * Reset mm_cid to its initial state (0).
         */
        if (put_user(mm_cid, &t->rseq->mm_cid))
                return -EFAULT;
        /*
         * Additional feature fields added after ORIG_RSEQ_SIZE
         * need to be conditionally reset only if
         * t->rseq_len != ORIG_RSEQ_SIZE.
         */
        return 0;
}

static int rseq_get_rseq_cs(struct task_struct *t, struct rseq_cs *rseq_cs)
{
        struct rseq_cs __user *urseq_cs;
        u64 ptr;
        u32 __user *usig;
        u32 sig;
        int ret;

#ifdef CONFIG_64BIT
        if (get_user(ptr, &t->rseq->rseq_cs))
                return -EFAULT;
#else
        if (copy_from_user(&ptr, &t->rseq->rseq_cs, sizeof(ptr)))
                return -EFAULT;
#endif
        if (!ptr) {
                memset(rseq_cs, 0, sizeof(*rseq_cs));
                return 0;
        }
        if (ptr >= TASK_SIZE)
                return -EINVAL;
        urseq_cs = (struct rseq_cs __user *)(unsigned long)ptr;
        if (copy_from_user(rseq_cs, urseq_cs, sizeof(*rseq_cs)))
                return -EFAULT;

        if (rseq_cs->start_ip >= TASK_SIZE ||
            rseq_cs->start_ip + rseq_cs->post_commit_offset >= TASK_SIZE ||
            rseq_cs->abort_ip >= TASK_SIZE ||
            rseq_cs->version > 0)
                return -EINVAL;
        /* Check for overflow. */
        if (rseq_cs->start_ip + rseq_cs->post_commit_offset < rseq_cs->start_ip)
                return -EINVAL;
        /* Ensure that abort_ip is not in the critical section. */
        if (rseq_cs->abort_ip - rseq_cs->start_ip < rseq_cs->post_commit_offset)
                return -EINVAL;

        usig = (u32 __user *)(unsigned long)(rseq_cs->abort_ip - sizeof(u32));
        ret = get_user(sig, usig);
        if (ret)
                return ret;

        if (current->rseq_sig != sig) {
                printk_ratelimited(KERN_WARNING
                        "Possible attack attempt. Unexpected rseq signature 0x%x, expecting 0x%x (pid=%d, addr=%p).\n",
                        sig, current->rseq_sig, current->pid, usig);
                return -EINVAL;
        }
        return 0;
}

static bool rseq_warn_flags(const char *str, u32 flags)
{
        u32 test_flags;

        if (!flags)
                return false;
        test_flags = flags & RSEQ_CS_NO_RESTART_FLAGS;
        if (test_flags)
                pr_warn_once("Deprecated flags (%u) in %s ABI structure", test_flags, str);
        test_flags = flags & ~RSEQ_CS_NO_RESTART_FLAGS;
        if (test_flags)
                pr_warn_once("Unknown flags (%u) in %s ABI structure", test_flags, str);
        return true;
}

static int rseq_need_restart(struct task_struct *t, u32 cs_flags)
{
        u32 flags, event_mask;
        int ret;

        if (rseq_warn_flags("rseq_cs", cs_flags))
                return -EINVAL;

        /* Get thread flags. */
        ret = get_user(flags, &t->rseq->flags);
        if (ret)
                return ret;

        if (rseq_warn_flags("rseq", flags))
                return -EINVAL;

        /*
         * Load and clear event mask atomically with respect to
         * scheduler preemption.
         */
        preempt_disable();
        event_mask = t->rseq_event_mask;
        t->rseq_event_mask = 0;
        preempt_enable();

        return !!event_mask;
}

static int clear_rseq_cs(struct task_struct *t)
{
        /*
         * The rseq_cs field is set to NULL on preemption or signal
         * delivery on top of rseq assembly block, as well as on top
         * of code outside of the rseq assembly block. This performs
         * a lazy clear of the rseq_cs field.
         *
         * Set rseq_cs to NULL.
         */
#ifdef CONFIG_64BIT
        return put_user(0UL, &t->rseq->rseq_cs);
#else
        if (clear_user(&t->rseq->rseq_cs, sizeof(t->rseq->rseq_cs)))
                return -EFAULT;
        return 0;
#endif
}

/*
 * Unsigned comparison will be true when ip >= start_ip, and when
 * ip < start_ip + post_commit_offset.
 */
static bool in_rseq_cs(unsigned long ip, struct rseq_cs *rseq_cs)
{
        return ip - rseq_cs->start_ip < rseq_cs->post_commit_offset;
}

static int rseq_ip_fixup(struct pt_regs *regs)
{
        unsigned long ip = instruction_pointer(regs);
        struct task_struct *t = current;
        struct rseq_cs rseq_cs;
        int ret;

        ret = rseq_get_rseq_cs(t, &rseq_cs);
        if (ret)
                return ret;

        /*
         * Handle potentially not being within a critical section.
         * If not nested over a rseq critical section, restart is useless.
         * Clear the rseq_cs pointer and return.
         */
        if (!in_rseq_cs(ip, &rseq_cs))
                return clear_rseq_cs(t);
        ret = rseq_need_restart(t, rseq_cs.flags);
        if (ret <= 0)
                return ret;
        ret = clear_rseq_cs(t);
        if (ret)
                return ret;
        trace_rseq_ip_fixup(ip, rseq_cs.start_ip, rseq_cs.post_commit_offset,
                            rseq_cs.abort_ip);
        instruction_pointer_set(regs, (unsigned long)rseq_cs.abort_ip);
        return 0;
}

/*
 * This resume handler must always be executed between any of:
 * - preemption,
 * - signal delivery,
 * and return to user-space.
 *
 * This is how we can ensure that the entire rseq critical section
 * will issue the commit instruction only if executed atomically with
 * respect to other threads scheduled on the same CPU, and with respect
 * to signal handlers.
 */
void __rseq_handle_notify_resume(struct ksignal *ksig, struct pt_regs *regs)
{
        struct task_struct *t = current;
        int ret, sig;

        if (unlikely(t->flags & PF_EXITING))
                return;

        /*
         * regs is NULL if and only if the caller is in a syscall path.  Skip
         * fixup and leave rseq_cs as is so that rseq_sycall() will detect and
         * kill a misbehaving userspace on debug kernels.
         */
        if (regs) {
                ret = rseq_ip_fixup(regs);
                if (unlikely(ret < 0))
                        goto error;
        }
        if (unlikely(rseq_update_cpu_node_id(t)))
                goto error;
        return;

error:
        sig = ksig ? ksig->sig : 0;
        force_sigsegv(sig);
}

#ifdef CONFIG_DEBUG_RSEQ

/*
 * Terminate the process if a syscall is issued within a restartable
 * sequence.
 */
void rseq_syscall(struct pt_regs *regs)
{
        unsigned long ip = instruction_pointer(regs);
        struct task_struct *t = current;
        struct rseq_cs rseq_cs;

        if (!t->rseq)
                return;
        if (rseq_get_rseq_cs(t, &rseq_cs) || in_rseq_cs(ip, &rseq_cs))
                force_sig(SIGSEGV);
}

#endif

/*
 * sys_rseq - setup restartable sequences for caller thread.
 */
SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len,
                int, flags, u32, sig)
{
        int ret;

        if (flags & RSEQ_FLAG_UNREGISTER) {
                if (flags & ~RSEQ_FLAG_UNREGISTER)
                        return -EINVAL;
                /* Unregister rseq for current thread. */
                if (current->rseq != rseq || !current->rseq)
                        return -EINVAL;
                if (rseq_len != current->rseq_len)
                        return -EINVAL;
                if (current->rseq_sig != sig)
                        return -EPERM;
                ret = rseq_reset_rseq_cpu_node_id(current);
                if (ret)
                        return ret;
                current->rseq = NULL;
                current->rseq_sig = 0;
                current->rseq_len = 0;
                return 0;
        }

        if (unlikely(flags))
                return -EINVAL;

        if (current->rseq) {
                /*
                 * If rseq is already registered, check whether
                 * the provided address differs from the prior
                 * one.
                 */
                if (current->rseq != rseq || rseq_len != current->rseq_len)
                        return -EINVAL;
                if (current->rseq_sig != sig)
                        return -EPERM;
                /* Already registered. */
                return -EBUSY;
        }

        /*
         * If there was no rseq previously registered, ensure the provided rseq
         * is properly aligned, as communcated to user-space through the ELF
         * auxiliary vector AT_RSEQ_ALIGN. If rseq_len is the original rseq
         * size, the required alignment is the original struct rseq alignment.
         *
         * In order to be valid, rseq_len is either the original rseq size, or
         * large enough to contain all supported fields, as communicated to
         * user-space through the ELF auxiliary vector AT_RSEQ_FEATURE_SIZE.
         */
        if (rseq_len < ORIG_RSEQ_SIZE ||
            (rseq_len == ORIG_RSEQ_SIZE && !IS_ALIGNED((unsigned long)rseq, ORIG_RSEQ_SIZE)) ||
            (rseq_len != ORIG_RSEQ_SIZE && (!IS_ALIGNED((unsigned long)rseq, __alignof__(*rseq)) ||
                                            rseq_len < offsetof(struct rseq, end))))
                return -EINVAL;
        if (!access_ok(rseq, rseq_len))
                return -EFAULT;
        current->rseq = rseq;
        current->rseq_len = rseq_len;
        current->rseq_sig = sig;
        /*
         * If rseq was previously inactive, and has just been
         * registered, ensure the cpu_id_start and cpu_id fields
         * are updated before returning to user-space.
         */
        rseq_set_notify_resume(current);

        return 0;
}






















































































































































































































































































   23 

   23 






   23 

   23 

   23 

   23 








   23 


   23 
   23 




   23 

   23 


    1 







    1 


    1 








    1 


    1 



    1 






    1 








    1 

    1 



    1 




    1 


    1 




    1 



    1 



    1 

    1 



    1 


    1 




    1 










    2 




    2 

    2 

    2 

























    2 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
// SPDX-License-Identifier: GPL-2.0

#include "bcachefs.h"

#include "acl.h"
#include "xattr.h"

#include <linux/posix_acl.h>

static const char * const acl_types[] = {
        [ACL_USER_OBJ]        = "user_obj",
        [ACL_USER]        = "user",
        [ACL_GROUP_OBJ]        = "group_obj",
        [ACL_GROUP]        = "group",
        [ACL_MASK]        = "mask",
        [ACL_OTHER]        = "other",
        NULL,
};

void bch2_acl_to_text(struct printbuf *out, const void *value, size_t size)
{
        const void *p, *end = value + size;

        if (!value ||
            size < sizeof(bch_acl_header) ||
            ((bch_acl_header *)value)->a_version != cpu_to_le32(BCH_ACL_VERSION))
                return;

        p = value + sizeof(bch_acl_header);
        while (p < end) {
                const bch_acl_entry *in = p;
                unsigned tag = le16_to_cpu(in->e_tag);

                prt_str(out, acl_types[tag]);

                switch (tag) {
                case ACL_USER_OBJ:
                case ACL_GROUP_OBJ:
                case ACL_MASK:
                case ACL_OTHER:
                        p += sizeof(bch_acl_entry_short);
                        break;
                case ACL_USER:
                        prt_printf(out, " uid %u", le32_to_cpu(in->e_id));
                        p += sizeof(bch_acl_entry);
                        break;
                case ACL_GROUP:
                        prt_printf(out, " gid %u", le32_to_cpu(in->e_id));
                        p += sizeof(bch_acl_entry);
                        break;
                }

                prt_printf(out, " %o", le16_to_cpu(in->e_perm));

                if (p != end)
                        prt_char(out, ' ');
        }
}

#ifdef CONFIG_BCACHEFS_POSIX_ACL

#include "fs.h"

#include <linux/fs.h>
#include <linux/posix_acl_xattr.h>
#include <linux/sched.h>
#include <linux/slab.h>

static inline size_t bch2_acl_size(unsigned nr_short, unsigned nr_long)
{
        return sizeof(bch_acl_header) +
                sizeof(bch_acl_entry_short) * nr_short +
                sizeof(bch_acl_entry) * nr_long;
}

static inline int acl_to_xattr_type(int type)
{
        switch (type) {
        case ACL_TYPE_ACCESS:
                return KEY_TYPE_XATTR_INDEX_POSIX_ACL_ACCESS;
        case ACL_TYPE_DEFAULT:
                return KEY_TYPE_XATTR_INDEX_POSIX_ACL_DEFAULT;
        default:
                BUG();
        }
}

/*
 * Convert from filesystem to in-memory representation.
 */
static struct posix_acl *bch2_acl_from_disk(struct btree_trans *trans,
                                            const void *value, size_t size)
{
        const void *p, *end = value + size;
        struct posix_acl *acl;
        struct posix_acl_entry *out;
        unsigned count = 0;
        int ret;

        if (!value)
                return NULL;
        if (size < sizeof(bch_acl_header))
                goto invalid;
        if (((bch_acl_header *)value)->a_version !=
            cpu_to_le32(BCH_ACL_VERSION))
                goto invalid;

        p = value + sizeof(bch_acl_header);
        while (p < end) {
                const bch_acl_entry *entry = p;

                if (p + sizeof(bch_acl_entry_short) > end)
                        goto invalid;

                switch (le16_to_cpu(entry->e_tag)) {
                case ACL_USER_OBJ:
                case ACL_GROUP_OBJ:
                case ACL_MASK:
                case ACL_OTHER:
                        p += sizeof(bch_acl_entry_short);
                        break;
                case ACL_USER:
                case ACL_GROUP:
                        p += sizeof(bch_acl_entry);
                        break;
                default:
                        goto invalid;
                }

                count++;
        }

        if (p > end)
                goto invalid;

        if (!count)
                return NULL;

        acl = allocate_dropping_locks(trans, ret,
                        posix_acl_alloc(count, _gfp));
        if (!acl)
                return ERR_PTR(-ENOMEM);
        if (ret) {
                kfree(acl);
                return ERR_PTR(ret);
        }

        out = acl->a_entries;

        p = value + sizeof(bch_acl_header);
        while (p < end) {
                const bch_acl_entry *in = p;

                out->e_tag  = le16_to_cpu(in->e_tag);
                out->e_perm = le16_to_cpu(in->e_perm);

                switch (out->e_tag) {
                case ACL_USER_OBJ:
                case ACL_GROUP_OBJ:
                case ACL_MASK:
                case ACL_OTHER:
                        p += sizeof(bch_acl_entry_short);
                        break;
                case ACL_USER:
                        out->e_uid = make_kuid(&init_user_ns,
                                               le32_to_cpu(in->e_id));
                        p += sizeof(bch_acl_entry);
                        break;
                case ACL_GROUP:
                        out->e_gid = make_kgid(&init_user_ns,
                                               le32_to_cpu(in->e_id));
                        p += sizeof(bch_acl_entry);
                        break;
                }

                out++;
        }

        BUG_ON(out != acl->a_entries + acl->a_count);

        return acl;
invalid:
        pr_err("invalid acl entry");
        return ERR_PTR(-EINVAL);
}

#define acl_for_each_entry(acl, acl_e)                        \
        for (acl_e = acl->a_entries;                        \
             acl_e < acl->a_entries + acl->a_count;        \
             acl_e++)

/*
 * Convert from in-memory to filesystem representation.
 */
static struct bkey_i_xattr *
bch2_acl_to_xattr(struct btree_trans *trans,
                  const struct posix_acl *acl,
                  int type)
{
        struct bkey_i_xattr *xattr;
        bch_acl_header *acl_header;
        const struct posix_acl_entry *acl_e;
        void *outptr;
        unsigned nr_short = 0, nr_long = 0, acl_len, u64s;

        acl_for_each_entry(acl, acl_e) {
                switch (acl_e->e_tag) {
                case ACL_USER:
                case ACL_GROUP:
                        nr_long++;
                        break;
                case ACL_USER_OBJ:
                case ACL_GROUP_OBJ:
                case ACL_MASK:
                case ACL_OTHER:
                        nr_short++;
                        break;
                default:
                        return ERR_PTR(-EINVAL);
                }
        }

        acl_len = bch2_acl_size(nr_short, nr_long);
        u64s = BKEY_U64s + xattr_val_u64s(0, acl_len);

        if (u64s > U8_MAX)
                return ERR_PTR(-E2BIG);

        xattr = bch2_trans_kmalloc(trans, u64s * sizeof(u64));
        if (IS_ERR(xattr))
                return xattr;

        bkey_xattr_init(&xattr->k_i);
        xattr->k.u64s                = u64s;
        xattr->v.x_type                = acl_to_xattr_type(type);
        xattr->v.x_name_len        = 0;
        xattr->v.x_val_len        = cpu_to_le16(acl_len);

        acl_header = xattr_val(&xattr->v);
        acl_header->a_version = cpu_to_le32(BCH_ACL_VERSION);

        outptr = (void *) acl_header + sizeof(*acl_header);

        acl_for_each_entry(acl, acl_e) {
                bch_acl_entry *entry = outptr;

                entry->e_tag = cpu_to_le16(acl_e->e_tag);
                entry->e_perm = cpu_to_le16(acl_e->e_perm);
                switch (acl_e->e_tag) {
                case ACL_USER:
                        entry->e_id = cpu_to_le32(
                                from_kuid(&init_user_ns, acl_e->e_uid));
                        outptr += sizeof(bch_acl_entry);
                        break;
                case ACL_GROUP:
                        entry->e_id = cpu_to_le32(
                                from_kgid(&init_user_ns, acl_e->e_gid));
                        outptr += sizeof(bch_acl_entry);
                        break;

                case ACL_USER_OBJ:
                case ACL_GROUP_OBJ:
                case ACL_MASK:
                case ACL_OTHER:
                        outptr += sizeof(bch_acl_entry_short);
                        break;
                }
        }

        BUG_ON(outptr != xattr_val(&xattr->v) + acl_len);

        return xattr;
}

struct posix_acl *bch2_get_acl(struct inode *vinode, int type, bool rcu)
{
        struct bch_inode_info *inode = to_bch_ei(vinode);
        struct bch_fs *c = inode->v.i_sb->s_fs_info;
        struct bch_hash_info hash = bch2_hash_info_init(c, &inode->ei_inode);
        struct xattr_search_key search = X_SEARCH(acl_to_xattr_type(type), "", 0);
        struct btree_iter iter = { NULL };
        struct posix_acl *acl = NULL;

        if (rcu)
                return ERR_PTR(-ECHILD);

        struct btree_trans *trans = bch2_trans_get(c);
retry:
        bch2_trans_begin(trans);

        struct bkey_s_c k = bch2_hash_lookup(trans, &iter, bch2_xattr_hash_desc,
                                             &hash, inode_inum(inode), &search, 0);
        int ret = bkey_err(k);
        if (ret)
                goto err;

        struct bkey_s_c_xattr xattr = bkey_s_c_to_xattr(k);
        acl = bch2_acl_from_disk(trans, xattr_val(xattr.v),
                                 le16_to_cpu(xattr.v->x_val_len));
        ret = PTR_ERR_OR_ZERO(acl);
err:
        if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
                goto retry;

        if (ret)
                acl = !bch2_err_matches(ret, ENOENT) ? ERR_PTR(ret) : NULL;

        if (!IS_ERR_OR_NULL(acl))
                set_cached_acl(&inode->v, type, acl);

        bch2_trans_iter_exit(trans, &iter);
        bch2_trans_put(trans);
        return acl;
}

int bch2_set_acl_trans(struct btree_trans *trans, subvol_inum inum,
                       struct bch_inode_unpacked *inode_u,
                       struct posix_acl *acl, int type)
{
        struct bch_hash_info hash_info = bch2_hash_info_init(trans->c, inode_u);
        int ret;

        if (type == ACL_TYPE_DEFAULT &&
            !S_ISDIR(inode_u->bi_mode))
                return acl ? -EACCES : 0;

        if (acl) {
                struct bkey_i_xattr *xattr =
                        bch2_acl_to_xattr(trans, acl, type);
                if (IS_ERR(xattr))
                        return PTR_ERR(xattr);

                ret = bch2_hash_set(trans, bch2_xattr_hash_desc, &hash_info,
                                    inum, &xattr->k_i, 0);
        } else {
                struct xattr_search_key search =
                        X_SEARCH(acl_to_xattr_type(type), "", 0);

                ret = bch2_hash_delete(trans, bch2_xattr_hash_desc, &hash_info,
                                       inum, &search);
        }

        return bch2_err_matches(ret, ENOENT) ? 0 : ret;
}

int bch2_set_acl(struct mnt_idmap *idmap,
                 struct dentry *dentry,
                 struct posix_acl *_acl, int type)
{
        struct bch_inode_info *inode = to_bch_ei(dentry->d_inode);
        struct bch_fs *c = inode->v.i_sb->s_fs_info;
        struct btree_iter inode_iter = { NULL };
        struct bch_inode_unpacked inode_u;
        struct posix_acl *acl;
        umode_t mode;
        int ret;

        mutex_lock(&inode->ei_update_lock);
        struct btree_trans *trans = bch2_trans_get(c);
retry:
        bch2_trans_begin(trans);
        acl = _acl;

        ret   = bch2_subvol_is_ro_trans(trans, inode->ei_inum.subvol) ?:
                bch2_inode_peek(trans, &inode_iter, &inode_u, inode_inum(inode),
                              BTREE_ITER_intent);
        if (ret)
                goto btree_err;

        mode = inode_u.bi_mode;

        if (type == ACL_TYPE_ACCESS) {
                ret = posix_acl_update_mode(idmap, &inode->v, &mode, &acl);
                if (ret)
                        goto btree_err;
        }

        ret = bch2_set_acl_trans(trans, inode_inum(inode), &inode_u, acl, type);
        if (ret)
                goto btree_err;

        inode_u.bi_ctime        = bch2_current_time(c);
        inode_u.bi_mode                = mode;

        ret =   bch2_inode_write(trans, &inode_iter, &inode_u) ?:
                bch2_trans_commit(trans, NULL, NULL, 0);
btree_err:
        bch2_trans_iter_exit(trans, &inode_iter);

        if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
                goto retry;
        if (unlikely(ret))
                goto err;

        bch2_inode_update_after_write(trans, inode, &inode_u,
                                      ATTR_CTIME|ATTR_MODE);

        set_cached_acl(&inode->v, type, acl);
err:
        bch2_trans_put(trans);
        mutex_unlock(&inode->ei_update_lock);

        return ret;
}

int bch2_acl_chmod(struct btree_trans *trans, subvol_inum inum,
                   struct bch_inode_unpacked *inode,
                   umode_t mode,
                   struct posix_acl **new_acl)
{
        struct bch_hash_info hash_info = bch2_hash_info_init(trans->c, inode);
        struct xattr_search_key search = X_SEARCH(KEY_TYPE_XATTR_INDEX_POSIX_ACL_ACCESS, "", 0);
        struct btree_iter iter;
        struct posix_acl *acl = NULL;

        struct bkey_s_c k = bch2_hash_lookup(trans, &iter, bch2_xattr_hash_desc,
                               &hash_info, inum, &search, BTREE_ITER_intent);
        int ret = bkey_err(k);
        if (ret)
                return bch2_err_matches(ret, ENOENT) ? 0 : ret;

        struct bkey_s_c_xattr xattr = bkey_s_c_to_xattr(k);

        acl = bch2_acl_from_disk(trans, xattr_val(xattr.v),
                        le16_to_cpu(xattr.v->x_val_len));
        ret = PTR_ERR_OR_ZERO(acl);
        if (ret)
                goto err;

        ret = allocate_dropping_locks_errcode(trans, __posix_acl_chmod(&acl, _gfp, mode));
        if (ret)
                goto err;

        struct bkey_i_xattr *new = bch2_acl_to_xattr(trans, acl, ACL_TYPE_ACCESS);
        ret = PTR_ERR_OR_ZERO(new);
        if (ret)
                goto err;

        new->k.p = iter.pos;
        ret = bch2_trans_update(trans, &iter, &new->k_i, 0);
        *new_acl = acl;
        acl = NULL;
err:
        bch2_trans_iter_exit(trans, &iter);
        if (!IS_ERR_OR_NULL(acl))
                kfree(acl);
        return ret;
}

#endif /* CONFIG_BCACHEFS_POSIX_ACL */



















    2 





    2 
















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/sch_blackhole.c        Black hole queue
 *
 * Authors:        Thomas Graf <tgraf@suug.ch>
 *
 * Note: Quantum tunneling is not supported.
 */

#include <linux/init.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <net/pkt_sched.h>

static int blackhole_enqueue(struct sk_buff *skb, struct Qdisc *sch,
                             struct sk_buff **to_free)
{
        qdisc_drop(skb, sch, to_free);
        return NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
}

static struct sk_buff *blackhole_dequeue(struct Qdisc *sch)
{
        return NULL;
}

static struct Qdisc_ops blackhole_qdisc_ops __read_mostly = {
        .id                = "blackhole",
        .priv_size        = 0,
        .enqueue        = blackhole_enqueue,
        .dequeue        = blackhole_dequeue,
        .peek                = blackhole_dequeue,
        .owner                = THIS_MODULE,
};

static int __init blackhole_init(void)
{
        return register_qdisc(&blackhole_qdisc_ops);
}
device_initcall(blackhole_init)


































  182 

  159 



  183 


















  144 










  130 
  144 
  125 
  144 
  125 
















  179 


















  180 

  178 


  179 
  180 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2000,2005 Silicon Graphics, Inc.
 * All Rights Reserved.
 */
#include "xfs.h"
#include "xfs_fs.h"
#include "xfs_shared.h"
#include "xfs_format.h"
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
#include "xfs_mount.h"
#include "xfs_inode.h"
#include "xfs_trans.h"
#include "xfs_trans_priv.h"
#include "xfs_inode_item.h"

#include <linux/iversion.h>

/*
 * Add a locked inode to the transaction.
 *
 * The inode must be locked, and it cannot be associated with any transaction.
 * If lock_flags is non-zero the inode will be unlocked on transaction commit.
 */
void
xfs_trans_ijoin(
        struct xfs_trans        *tp,
        struct xfs_inode        *ip,
        uint                        lock_flags)
{
        struct xfs_inode_log_item *iip;

        xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
        if (ip->i_itemp == NULL)
                xfs_inode_item_init(ip, ip->i_mount);
        iip = ip->i_itemp;

        ASSERT(iip->ili_lock_flags == 0);
        iip->ili_lock_flags = lock_flags;
        ASSERT(!xfs_iflags_test(ip, XFS_ISTALE));

        /* Reset the per-tx dirty context and add the item to the tx. */
        iip->ili_dirty_flags = 0;
        xfs_trans_add_item(tp, &iip->ili_item);
}

/*
 * Transactional inode timestamp update. Requires the inode to be locked and
 * joined to the transaction supplied. Relies on the transaction subsystem to
 * track dirty state and update/writeback the inode accordingly.
 */
void
xfs_trans_ichgtime(
        struct xfs_trans        *tp,
        struct xfs_inode        *ip,
        int                        flags)
{
        struct inode                *inode = VFS_I(ip);
        struct timespec64        tv;

        ASSERT(tp);
        xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);

        /* If the mtime changes, then ctime must also change */
        ASSERT(flags & XFS_ICHGTIME_CHG);

        tv = inode_set_ctime_current(inode);
        if (flags & XFS_ICHGTIME_MOD)
                inode_set_mtime_to_ts(inode, tv);
        if (flags & XFS_ICHGTIME_ACCESS)
                inode_set_atime_to_ts(inode, tv);
        if (flags & XFS_ICHGTIME_CREATE)
                ip->i_crtime = tv;
}

/*
 * This is called to mark the fields indicated in fieldmask as needing to be
 * logged when the transaction is committed.  The inode must already be
 * associated with the given transaction. All we do here is record where the
 * inode was dirtied and mark the transaction and inode log item dirty;
 * everything else is done in the ->precommit log item operation after the
 * changes in the transaction have been completed.
 */
void
xfs_trans_log_inode(
        struct xfs_trans        *tp,
        struct xfs_inode        *ip,
        uint                        flags)
{
        struct xfs_inode_log_item *iip = ip->i_itemp;
        struct inode                *inode = VFS_I(ip);

        ASSERT(iip);
        xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
        ASSERT(!xfs_iflags_test(ip, XFS_ISTALE));

        tp->t_flags |= XFS_TRANS_DIRTY;

        /*
         * First time we log the inode in a transaction, bump the inode change
         * counter if it is configured for this to occur. While we have the
         * inode locked exclusively for metadata modification, we can usually
         * avoid setting XFS_ILOG_CORE if no one has queried the value since
         * the last time it was incremented. If we have XFS_ILOG_CORE already
         * set however, then go ahead and bump the i_version counter
         * unconditionally.
         */
        if (!test_and_set_bit(XFS_LI_DIRTY, &iip->ili_item.li_flags)) {
                if (IS_I_VERSION(inode) &&
                    inode_maybe_inc_iversion(inode, flags & XFS_ILOG_CORE))
                        flags |= XFS_ILOG_IVERSION;
        }

        iip->ili_dirty_flags |= flags;
}

int
xfs_trans_roll_inode(
        struct xfs_trans        **tpp,
        struct xfs_inode        *ip)
{
        int                        error;

        xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE);
        error = xfs_trans_roll(tpp);
        if (!error)
                xfs_trans_ijoin(*tpp, ip, 0);
        return error;
}






































































































































































































































    6 

    6 



    6 


    6 


    6 





    6 

















    6 







    6 



















    6 
    6 
    6 


    6 

    6 

    6 














    6 


    6 







































    6 


    6 




























































































































































































    6 

































    6 


    6 

    6 















































































































    6 


    6 



    6 



    6 







    6 



    6 











    6 
    6 









    6 














    6 





    6 
    6 












    6 












    6 
    6 
    6 
    6 








    6 
    6 
    6 
    6 


    6 



    6 













    7 

    7 
    7 














































  132 













  132 

  132 

  134 
















  132 





  127 


  133 
  127 




  133 


  134 

  134 
  128 
  135 
















  135 




    7 



  134 





































































































































































































































































































































































































































































































































































































































































































    7 




    7 






    7 







    7 







    7 






    7 




    7 





    7 

    7 






























































































































































































































































































































































































































































































































































































































































































































































































































































































































    6 




    6 




















    6 



























































































































































































































































































































































































































































































    6 















   14 












   15 







   15 


   14 
   14 
   14 






















   18 







   17 

   16 



   16 

    5 

    5 

    5 
    5 



    4 


   15 









   12 
   12 







   15 





   14 
   15 

   15 

   15 








   15 

    3 
    2 
    3 



    3 







    2 
    3 







    1 
    1 

    1 



    2 



   12 
   12 


   14 



   14 
   14 
   14 






    1 






   16 













































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
/*
 * Copyright (c) 2016 Intel Corporation
 *
 * Permission to use, copy, modify, distribute, and sell this software and its
 * documentation for any purpose is hereby granted without fee, provided that
 * the above copyright notice appear in all copies and that both that copyright
 * notice and this permission notice appear in supporting documentation, and
 * that the name of the copyright holders not be used in advertising or
 * publicity pertaining to distribution of the software without specific,
 * written prior permission.  The copyright holders make no representations
 * about the suitability of this software for any purpose.  It is provided "as
 * is" without express or implied warranty.
 *
 * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
 * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
 * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
 * OF THIS SOFTWARE.
 */

#include <drm/drm_auth.h>
#include <drm/drm_connector.h>
#include <drm/drm_drv.h>
#include <drm/drm_edid.h>
#include <drm/drm_encoder.h>
#include <drm/drm_file.h>
#include <drm/drm_managed.h>
#include <drm/drm_panel.h>
#include <drm/drm_print.h>
#include <drm/drm_privacy_screen_consumer.h>
#include <drm/drm_sysfs.h>
#include <drm/drm_utils.h>

#include <linux/property.h>
#include <linux/uaccess.h>

#include <video/cmdline.h>

#include "drm_crtc_internal.h"
#include "drm_internal.h"

/**
 * DOC: overview
 *
 * In DRM connectors are the general abstraction for display sinks, and include
 * also fixed panels or anything else that can display pixels in some form. As
 * opposed to all other KMS objects representing hardware (like CRTC, encoder or
 * plane abstractions) connectors can be hotplugged and unplugged at runtime.
 * Hence they are reference-counted using drm_connector_get() and
 * drm_connector_put().
 *
 * KMS driver must create, initialize, register and attach at a &struct
 * drm_connector for each such sink. The instance is created as other KMS
 * objects and initialized by setting the following fields. The connector is
 * initialized with a call to drm_connector_init() with a pointer to the
 * &struct drm_connector_funcs and a connector type, and then exposed to
 * userspace with a call to drm_connector_register().
 *
 * Connectors must be attached to an encoder to be used. For devices that map
 * connectors to encoders 1:1, the connector should be attached at
 * initialization time with a call to drm_connector_attach_encoder(). The
 * driver must also set the &drm_connector.encoder field to point to the
 * attached encoder.
 *
 * For connectors which are not fixed (like built-in panels) the driver needs to
 * support hotplug notifications. The simplest way to do that is by using the
 * probe helpers, see drm_kms_helper_poll_init() for connectors which don't have
 * hardware support for hotplug interrupts. Connectors with hardware hotplug
 * support can instead use e.g. drm_helper_hpd_irq_event().
 */

/*
 * Global connector list for drm_connector_find_by_fwnode().
 * Note drm_connector_[un]register() first take connector->lock and then
 * take the connector_list_lock.
 */
static DEFINE_MUTEX(connector_list_lock);
static LIST_HEAD(connector_list);

struct drm_conn_prop_enum_list {
        int type;
        const char *name;
        struct ida ida;
};

/*
 * Connector and encoder types.
 */
static struct drm_conn_prop_enum_list drm_connector_enum_list[] = {
        { DRM_MODE_CONNECTOR_Unknown, "Unknown" },
        { DRM_MODE_CONNECTOR_VGA, "VGA" },
        { DRM_MODE_CONNECTOR_DVII, "DVI-I" },
        { DRM_MODE_CONNECTOR_DVID, "DVI-D" },
        { DRM_MODE_CONNECTOR_DVIA, "DVI-A" },
        { DRM_MODE_CONNECTOR_Composite, "Composite" },
        { DRM_MODE_CONNECTOR_SVIDEO, "SVIDEO" },
        { DRM_MODE_CONNECTOR_LVDS, "LVDS" },
        { DRM_MODE_CONNECTOR_Component, "Component" },
        { DRM_MODE_CONNECTOR_9PinDIN, "DIN" },
        { DRM_MODE_CONNECTOR_DisplayPort, "DP" },
        { DRM_MODE_CONNECTOR_HDMIA, "HDMI-A" },
        { DRM_MODE_CONNECTOR_HDMIB, "HDMI-B" },
        { DRM_MODE_CONNECTOR_TV, "TV" },
        { DRM_MODE_CONNECTOR_eDP, "eDP" },
        { DRM_MODE_CONNECTOR_VIRTUAL, "Virtual" },
        { DRM_MODE_CONNECTOR_DSI, "DSI" },
        { DRM_MODE_CONNECTOR_DPI, "DPI" },
        { DRM_MODE_CONNECTOR_WRITEBACK, "Writeback" },
        { DRM_MODE_CONNECTOR_SPI, "SPI" },
        { DRM_MODE_CONNECTOR_USB, "USB" },
};

void drm_connector_ida_init(void)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(drm_connector_enum_list); i++)
                ida_init(&drm_connector_enum_list[i].ida);
}

void drm_connector_ida_destroy(void)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(drm_connector_enum_list); i++)
                ida_destroy(&drm_connector_enum_list[i].ida);
}

/**
 * drm_get_connector_type_name - return a string for connector type
 * @type: The connector type (DRM_MODE_CONNECTOR_*)
 *
 * Returns: the name of the connector type, or NULL if the type is not valid.
 */
const char *drm_get_connector_type_name(unsigned int type)
{
        if (type < ARRAY_SIZE(drm_connector_enum_list))
                return drm_connector_enum_list[type].name;

        return NULL;
}
EXPORT_SYMBOL(drm_get_connector_type_name);

/**
 * drm_connector_get_cmdline_mode - reads the user's cmdline mode
 * @connector: connector to query
 *
 * The kernel supports per-connector configuration of its consoles through
 * use of the video= parameter. This function parses that option and
 * extracts the user's specified mode (or enable/disable status) for a
 * particular connector. This is typically only used during the early fbdev
 * setup.
 */
static void drm_connector_get_cmdline_mode(struct drm_connector *connector)
{
        struct drm_cmdline_mode *mode = &connector->cmdline_mode;
        const char *option;

        option = video_get_options(connector->name);
        if (!option)
                return;

        if (!drm_mode_parse_command_line_for_connector(option,
                                                       connector,
                                                       mode))
                return;

        if (mode->force) {
                DRM_INFO("forcing %s connector %s\n", connector->name,
                         drm_get_connector_force_name(mode->force));
                connector->force = mode->force;
        }

        if (mode->panel_orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) {
                DRM_INFO("cmdline forces connector %s panel_orientation to %d\n",
                         connector->name, mode->panel_orientation);
                drm_connector_set_panel_orientation(connector,
                                                    mode->panel_orientation);
        }

        DRM_DEBUG_KMS("cmdline mode for connector %s %s %dx%d@%dHz%s%s%s\n",
                      connector->name, mode->name,
                      mode->xres, mode->yres,
                      mode->refresh_specified ? mode->refresh : 60,
                      mode->rb ? " reduced blanking" : "",
                      mode->margins ? " with margins" : "",
                      mode->interlace ?  " interlaced" : "");
}

static void drm_connector_free(struct kref *kref)
{
        struct drm_connector *connector =
                container_of(kref, struct drm_connector, base.refcount);
        struct drm_device *dev = connector->dev;

        drm_mode_object_unregister(dev, &connector->base);
        connector->funcs->destroy(connector);
}

void drm_connector_free_work_fn(struct work_struct *work)
{
        struct drm_connector *connector, *n;
        struct drm_device *dev =
                container_of(work, struct drm_device, mode_config.connector_free_work);
        struct drm_mode_config *config = &dev->mode_config;
        unsigned long flags;
        struct llist_node *freed;

        spin_lock_irqsave(&config->connector_list_lock, flags);
        freed = llist_del_all(&config->connector_free_list);
        spin_unlock_irqrestore(&config->connector_list_lock, flags);

        llist_for_each_entry_safe(connector, n, freed, free_node) {
                drm_mode_object_unregister(dev, &connector->base);
                connector->funcs->destroy(connector);
        }
}

static int __drm_connector_init(struct drm_device *dev,
                                struct drm_connector *connector,
                                const struct drm_connector_funcs *funcs,
                                int connector_type,
                                struct i2c_adapter *ddc)
{
        struct drm_mode_config *config = &dev->mode_config;
        int ret;
        struct ida *connector_ida =
                &drm_connector_enum_list[connector_type].ida;

        WARN_ON(drm_drv_uses_atomic_modeset(dev) &&
                (!funcs->atomic_destroy_state ||
                 !funcs->atomic_duplicate_state));

        ret = __drm_mode_object_add(dev, &connector->base,
                                    DRM_MODE_OBJECT_CONNECTOR,
                                    false, drm_connector_free);
        if (ret)
                return ret;

        connector->base.properties = &connector->properties;
        connector->dev = dev;
        connector->funcs = funcs;

        /* connector index is used with 32bit bitmasks */
        ret = ida_alloc_max(&config->connector_ida, 31, GFP_KERNEL);
        if (ret < 0) {
                DRM_DEBUG_KMS("Failed to allocate %s connector index: %d\n",
                              drm_connector_enum_list[connector_type].name,
                              ret);
                goto out_put;
        }
        connector->index = ret;
        ret = 0;

        connector->connector_type = connector_type;
        connector->connector_type_id =
                ida_alloc_min(connector_ida, 1, GFP_KERNEL);
        if (connector->connector_type_id < 0) {
                ret = connector->connector_type_id;
                goto out_put_id;
        }
        connector->name =
                kasprintf(GFP_KERNEL, "%s-%d",
                          drm_connector_enum_list[connector_type].name,
                          connector->connector_type_id);
        if (!connector->name) {
                ret = -ENOMEM;
                goto out_put_type_id;
        }

        /* provide ddc symlink in sysfs */
        connector->ddc = ddc;

        INIT_LIST_HEAD(&connector->global_connector_list_entry);
        INIT_LIST_HEAD(&connector->probed_modes);
        INIT_LIST_HEAD(&connector->modes);
        mutex_init(&connector->mutex);
        mutex_init(&connector->edid_override_mutex);
        mutex_init(&connector->hdmi.infoframes.lock);
        connector->edid_blob_ptr = NULL;
        connector->epoch_counter = 0;
        connector->tile_blob_ptr = NULL;
        connector->status = connector_status_unknown;
        connector->display_info.panel_orientation =
                DRM_MODE_PANEL_ORIENTATION_UNKNOWN;

        drm_connector_get_cmdline_mode(connector);

        /* We should add connectors at the end to avoid upsetting the connector
         * index too much.
         */
        spin_lock_irq(&config->connector_list_lock);
        list_add_tail(&connector->head, &config->connector_list);
        config->num_connector++;
        spin_unlock_irq(&config->connector_list_lock);

        if (connector_type != DRM_MODE_CONNECTOR_VIRTUAL &&
            connector_type != DRM_MODE_CONNECTOR_WRITEBACK)
                drm_connector_attach_edid_property(connector);

        drm_object_attach_property(&connector->base,
                                      config->dpms_property, 0);

        drm_object_attach_property(&connector->base,
                                   config->link_status_property,
                                   0);

        drm_object_attach_property(&connector->base,
                                   config->non_desktop_property,
                                   0);
        drm_object_attach_property(&connector->base,
                                   config->tile_property,
                                   0);

        if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
                drm_object_attach_property(&connector->base, config->prop_crtc_id, 0);
        }

        connector->debugfs_entry = NULL;
out_put_type_id:
        if (ret)
                ida_free(connector_ida, connector->connector_type_id);
out_put_id:
        if (ret)
                ida_free(&config->connector_ida, connector->index);
out_put:
        if (ret)
                drm_mode_object_unregister(dev, &connector->base);

        return ret;
}

/**
 * drm_connector_init - Init a preallocated connector
 * @dev: DRM device
 * @connector: the connector to init
 * @funcs: callbacks for this connector
 * @connector_type: user visible type of the connector
 *
 * Initialises a preallocated connector. Connectors should be
 * subclassed as part of driver connector objects.
 *
 * At driver unload time the driver's &drm_connector_funcs.destroy hook
 * should call drm_connector_cleanup() and free the connector structure.
 * The connector structure should not be allocated with devm_kzalloc().
 *
 * Note: consider using drmm_connector_init() instead of
 * drm_connector_init() to let the DRM managed resource infrastructure
 * take care of cleanup and deallocation.
 *
 * Returns:
 * Zero on success, error code on failure.
 */
int drm_connector_init(struct drm_device *dev,
                       struct drm_connector *connector,
                       const struct drm_connector_funcs *funcs,
                       int connector_type)
{
        if (drm_WARN_ON(dev, !(funcs && funcs->destroy)))
                return -EINVAL;

        return __drm_connector_init(dev, connector, funcs, connector_type, NULL);
}
EXPORT_SYMBOL(drm_connector_init);

/**
 * drm_connector_init_with_ddc - Init a preallocated connector
 * @dev: DRM device
 * @connector: the connector to init
 * @funcs: callbacks for this connector
 * @connector_type: user visible type of the connector
 * @ddc: pointer to the associated ddc adapter
 *
 * Initialises a preallocated connector. Connectors should be
 * subclassed as part of driver connector objects.
 *
 * At driver unload time the driver's &drm_connector_funcs.destroy hook
 * should call drm_connector_cleanup() and free the connector structure.
 * The connector structure should not be allocated with devm_kzalloc().
 *
 * Ensures that the ddc field of the connector is correctly set.
 *
 * Note: consider using drmm_connector_init() instead of
 * drm_connector_init_with_ddc() to let the DRM managed resource
 * infrastructure take care of cleanup and deallocation.
 *
 * Returns:
 * Zero on success, error code on failure.
 */
int drm_connector_init_with_ddc(struct drm_device *dev,
                                struct drm_connector *connector,
                                const struct drm_connector_funcs *funcs,
                                int connector_type,
                                struct i2c_adapter *ddc)
{
        if (drm_WARN_ON(dev, !(funcs && funcs->destroy)))
                return -EINVAL;

        return __drm_connector_init(dev, connector, funcs, connector_type, ddc);
}
EXPORT_SYMBOL(drm_connector_init_with_ddc);

static void drm_connector_cleanup_action(struct drm_device *dev,
                                         void *ptr)
{
        struct drm_connector *connector = ptr;

        drm_connector_cleanup(connector);
}

/**
 * drmm_connector_init - Init a preallocated connector
 * @dev: DRM device
 * @connector: the connector to init
 * @funcs: callbacks for this connector
 * @connector_type: user visible type of the connector
 * @ddc: optional pointer to the associated ddc adapter
 *
 * Initialises a preallocated connector. Connectors should be
 * subclassed as part of driver connector objects.
 *
 * Cleanup is automatically handled with a call to
 * drm_connector_cleanup() in a DRM-managed action.
 *
 * The connector structure should be allocated with drmm_kzalloc().
 *
 * The @drm_connector_funcs.destroy hook must be NULL.
 *
 * Returns:
 * Zero on success, error code on failure.
 */
int drmm_connector_init(struct drm_device *dev,
                        struct drm_connector *connector,
                        const struct drm_connector_funcs *funcs,
                        int connector_type,
                        struct i2c_adapter *ddc)
{
        int ret;

        if (drm_WARN_ON(dev, funcs && funcs->destroy))
                return -EINVAL;

        ret = __drm_connector_init(dev, connector, funcs, connector_type, ddc);
        if (ret)
                return ret;

        ret = drmm_add_action_or_reset(dev, drm_connector_cleanup_action,
                                       connector);
        if (ret)
                return ret;

        return 0;
}
EXPORT_SYMBOL(drmm_connector_init);

/**
 * drmm_connector_hdmi_init - Init a preallocated HDMI connector
 * @dev: DRM device
 * @connector: A pointer to the HDMI connector to init
 * @vendor: HDMI Controller Vendor name
 * @product: HDMI Controller Product name
 * @funcs: callbacks for this connector
 * @hdmi_funcs: HDMI-related callbacks for this connector
 * @connector_type: user visible type of the connector
 * @ddc: optional pointer to the associated ddc adapter
 * @supported_formats: Bitmask of @hdmi_colorspace listing supported output formats
 * @max_bpc: Maximum bits per char the HDMI connector supports
 *
 * Initialises a preallocated HDMI connector. Connectors can be
 * subclassed as part of driver connector objects.
 *
 * Cleanup is automatically handled with a call to
 * drm_connector_cleanup() in a DRM-managed action.
 *
 * The connector structure should be allocated with drmm_kzalloc().
 *
 * The @drm_connector_funcs.destroy hook must be NULL.
 *
 * Returns:
 * Zero on success, error code on failure.
 */
int drmm_connector_hdmi_init(struct drm_device *dev,
                             struct drm_connector *connector,
                             const char *vendor, const char *product,
                             const struct drm_connector_funcs *funcs,
                             const struct drm_connector_hdmi_funcs *hdmi_funcs,
                             int connector_type,
                             struct i2c_adapter *ddc,
                             unsigned long supported_formats,
                             unsigned int max_bpc)
{
        int ret;

        if (!vendor || !product)
                return -EINVAL;

        if ((strlen(vendor) > DRM_CONNECTOR_HDMI_VENDOR_LEN) ||
            (strlen(product) > DRM_CONNECTOR_HDMI_PRODUCT_LEN))
                return -EINVAL;

        if (!(connector_type == DRM_MODE_CONNECTOR_HDMIA ||
              connector_type == DRM_MODE_CONNECTOR_HDMIB))
                return -EINVAL;

        if (!supported_formats || !(supported_formats & BIT(HDMI_COLORSPACE_RGB)))
                return -EINVAL;

        if (connector->ycbcr_420_allowed != !!(supported_formats & BIT(HDMI_COLORSPACE_YUV420)))
                return -EINVAL;

        if (!(max_bpc == 8 || max_bpc == 10 || max_bpc == 12))
                return -EINVAL;

        ret = drmm_connector_init(dev, connector, funcs, connector_type, ddc);
        if (ret)
                return ret;

        connector->hdmi.supported_formats = supported_formats;
        strtomem_pad(connector->hdmi.vendor, vendor, 0);
        strtomem_pad(connector->hdmi.product, product, 0);

        /*
         * drm_connector_attach_max_bpc_property() requires the
         * connector to have a state.
         */
        if (connector->funcs->reset)
                connector->funcs->reset(connector);

        drm_connector_attach_max_bpc_property(connector, 8, max_bpc);
        connector->max_bpc = max_bpc;

        if (max_bpc > 8)
                drm_connector_attach_hdr_output_metadata_property(connector);

        connector->hdmi.funcs = hdmi_funcs;

        return 0;
}
EXPORT_SYMBOL(drmm_connector_hdmi_init);

/**
 * drm_connector_attach_edid_property - attach edid property.
 * @connector: the connector
 *
 * Some connector types like DRM_MODE_CONNECTOR_VIRTUAL do not get a
 * edid property attached by default.  This function can be used to
 * explicitly enable the edid property in these cases.
 */
void drm_connector_attach_edid_property(struct drm_connector *connector)
{
        struct drm_mode_config *config = &connector->dev->mode_config;

        drm_object_attach_property(&connector->base,
                                   config->edid_property,
                                   0);
}
EXPORT_SYMBOL(drm_connector_attach_edid_property);

/**
 * drm_connector_attach_encoder - attach a connector to an encoder
 * @connector: connector to attach
 * @encoder: encoder to attach @connector to
 *
 * This function links up a connector to an encoder. Note that the routing
 * restrictions between encoders and crtcs are exposed to userspace through the
 * possible_clones and possible_crtcs bitmasks.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_attach_encoder(struct drm_connector *connector,
                                 struct drm_encoder *encoder)
{
        /*
         * In the past, drivers have attempted to model the static association
         * of connector to encoder in simple connector/encoder devices using a
         * direct assignment of connector->encoder = encoder. This connection
         * is a logical one and the responsibility of the core, so drivers are
         * expected not to mess with this.
         *
         * Note that the error return should've been enough here, but a large
         * majority of drivers ignores the return value, so add in a big WARN
         * to get people's attention.
         */
        if (WARN_ON(connector->encoder))
                return -EINVAL;

        connector->possible_encoders |= drm_encoder_mask(encoder);

        return 0;
}
EXPORT_SYMBOL(drm_connector_attach_encoder);

/**
 * drm_connector_has_possible_encoder - check if the connector and encoder are
 * associated with each other
 * @connector: the connector
 * @encoder: the encoder
 *
 * Returns:
 * True if @encoder is one of the possible encoders for @connector.
 */
bool drm_connector_has_possible_encoder(struct drm_connector *connector,
                                        struct drm_encoder *encoder)
{
        return connector->possible_encoders & drm_encoder_mask(encoder);
}
EXPORT_SYMBOL(drm_connector_has_possible_encoder);

static void drm_mode_remove(struct drm_connector *connector,
                            struct drm_display_mode *mode)
{
        list_del(&mode->head);
        drm_mode_destroy(connector->dev, mode);
}

/**
 * drm_connector_cleanup - cleans up an initialised connector
 * @connector: connector to cleanup
 *
 * Cleans up the connector but doesn't free the object.
 */
void drm_connector_cleanup(struct drm_connector *connector)
{
        struct drm_device *dev = connector->dev;
        struct drm_display_mode *mode, *t;

        /* The connector should have been removed from userspace long before
         * it is finally destroyed.
         */
        if (WARN_ON(connector->registration_state ==
                    DRM_CONNECTOR_REGISTERED))
                drm_connector_unregister(connector);

        if (connector->privacy_screen) {
                drm_privacy_screen_put(connector->privacy_screen);
                connector->privacy_screen = NULL;
        }

        if (connector->tile_group) {
                drm_mode_put_tile_group(dev, connector->tile_group);
                connector->tile_group = NULL;
        }

        list_for_each_entry_safe(mode, t, &connector->probed_modes, head)
                drm_mode_remove(connector, mode);

        list_for_each_entry_safe(mode, t, &connector->modes, head)
                drm_mode_remove(connector, mode);

        ida_free(&drm_connector_enum_list[connector->connector_type].ida,
                          connector->connector_type_id);

        ida_free(&dev->mode_config.connector_ida, connector->index);

        kfree(connector->display_info.bus_formats);
        kfree(connector->display_info.vics);
        drm_mode_object_unregister(dev, &connector->base);
        kfree(connector->name);
        connector->name = NULL;
        fwnode_handle_put(connector->fwnode);
        connector->fwnode = NULL;
        spin_lock_irq(&dev->mode_config.connector_list_lock);
        list_del(&connector->head);
        dev->mode_config.num_connector--;
        spin_unlock_irq(&dev->mode_config.connector_list_lock);

        WARN_ON(connector->state && !connector->funcs->atomic_destroy_state);
        if (connector->state && connector->funcs->atomic_destroy_state)
                connector->funcs->atomic_destroy_state(connector,
                                                       connector->state);

        mutex_destroy(&connector->hdmi.infoframes.lock);
        mutex_destroy(&connector->mutex);

        memset(connector, 0, sizeof(*connector));

        if (dev->registered)
                drm_sysfs_hotplug_event(dev);
}
EXPORT_SYMBOL(drm_connector_cleanup);

/**
 * drm_connector_register - register a connector
 * @connector: the connector to register
 *
 * Register userspace interfaces for a connector. Only call this for connectors
 * which can be hotplugged after drm_dev_register() has been called already,
 * e.g. DP MST connectors. All other connectors will be registered automatically
 * when calling drm_dev_register().
 *
 * When the connector is no longer available, callers must call
 * drm_connector_unregister().
 *
 * Returns:
 * Zero on success, error code on failure.
 */
int drm_connector_register(struct drm_connector *connector)
{
        int ret = 0;

        if (!connector->dev->registered)
                return 0;

        mutex_lock(&connector->mutex);
        if (connector->registration_state != DRM_CONNECTOR_INITIALIZING)
                goto unlock;

        ret = drm_sysfs_connector_add(connector);
        if (ret)
                goto unlock;

        drm_debugfs_connector_add(connector);

        if (connector->funcs->late_register) {
                ret = connector->funcs->late_register(connector);
                if (ret)
                        goto err_debugfs;
        }

        ret = drm_sysfs_connector_add_late(connector);
        if (ret)
                goto err_late_register;

        drm_mode_object_register(connector->dev, &connector->base);

        connector->registration_state = DRM_CONNECTOR_REGISTERED;

        /* Let userspace know we have a new connector */
        drm_sysfs_connector_hotplug_event(connector);

        if (connector->privacy_screen)
                drm_privacy_screen_register_notifier(connector->privacy_screen,
                                           &connector->privacy_screen_notifier);

        mutex_lock(&connector_list_lock);
        list_add_tail(&connector->global_connector_list_entry, &connector_list);
        mutex_unlock(&connector_list_lock);
        goto unlock;

err_late_register:
        if (connector->funcs->early_unregister)
                connector->funcs->early_unregister(connector);
err_debugfs:
        drm_debugfs_connector_remove(connector);
        drm_sysfs_connector_remove(connector);
unlock:
        mutex_unlock(&connector->mutex);
        return ret;
}
EXPORT_SYMBOL(drm_connector_register);

/**
 * drm_connector_unregister - unregister a connector
 * @connector: the connector to unregister
 *
 * Unregister userspace interfaces for a connector. Only call this for
 * connectors which have been registered explicitly by calling
 * drm_connector_register().
 */
void drm_connector_unregister(struct drm_connector *connector)
{
        mutex_lock(&connector->mutex);
        if (connector->registration_state != DRM_CONNECTOR_REGISTERED) {
                mutex_unlock(&connector->mutex);
                return;
        }

        mutex_lock(&connector_list_lock);
        list_del_init(&connector->global_connector_list_entry);
        mutex_unlock(&connector_list_lock);

        if (connector->privacy_screen)
                drm_privacy_screen_unregister_notifier(
                                        connector->privacy_screen,
                                        &connector->privacy_screen_notifier);

        drm_sysfs_connector_remove_early(connector);

        if (connector->funcs->early_unregister)
                connector->funcs->early_unregister(connector);

        drm_debugfs_connector_remove(connector);
        drm_sysfs_connector_remove(connector);

        connector->registration_state = DRM_CONNECTOR_UNREGISTERED;
        mutex_unlock(&connector->mutex);
}
EXPORT_SYMBOL(drm_connector_unregister);

void drm_connector_unregister_all(struct drm_device *dev)
{
        struct drm_connector *connector;
        struct drm_connector_list_iter conn_iter;

        drm_connector_list_iter_begin(dev, &conn_iter);
        drm_for_each_connector_iter(connector, &conn_iter)
                drm_connector_unregister(connector);
        drm_connector_list_iter_end(&conn_iter);
}

int drm_connector_register_all(struct drm_device *dev)
{
        struct drm_connector *connector;
        struct drm_connector_list_iter conn_iter;
        int ret = 0;

        drm_connector_list_iter_begin(dev, &conn_iter);
        drm_for_each_connector_iter(connector, &conn_iter) {
                ret = drm_connector_register(connector);
                if (ret)
                        break;
        }
        drm_connector_list_iter_end(&conn_iter);

        if (ret)
                drm_connector_unregister_all(dev);
        return ret;
}

/**
 * drm_get_connector_status_name - return a string for connector status
 * @status: connector status to compute name of
 *
 * In contrast to the other drm_get_*_name functions this one here returns a
 * const pointer and hence is threadsafe.
 *
 * Returns: connector status string
 */
const char *drm_get_connector_status_name(enum drm_connector_status status)
{
        if (status == connector_status_connected)
                return "connected";
        else if (status == connector_status_disconnected)
                return "disconnected";
        else
                return "unknown";
}
EXPORT_SYMBOL(drm_get_connector_status_name);

/**
 * drm_get_connector_force_name - return a string for connector force
 * @force: connector force to get name of
 *
 * Returns: const pointer to name.
 */
const char *drm_get_connector_force_name(enum drm_connector_force force)
{
        switch (force) {
        case DRM_FORCE_UNSPECIFIED:
                return "unspecified";
        case DRM_FORCE_OFF:
                return "off";
        case DRM_FORCE_ON:
                return "on";
        case DRM_FORCE_ON_DIGITAL:
                return "digital";
        default:
                return "unknown";
        }
}

#ifdef CONFIG_LOCKDEP
static struct lockdep_map connector_list_iter_dep_map = {
        .name = "drm_connector_list_iter"
};
#endif

/**
 * drm_connector_list_iter_begin - initialize a connector_list iterator
 * @dev: DRM device
 * @iter: connector_list iterator
 *
 * Sets @iter up to walk the &drm_mode_config.connector_list of @dev. @iter
 * must always be cleaned up again by calling drm_connector_list_iter_end().
 * Iteration itself happens using drm_connector_list_iter_next() or
 * drm_for_each_connector_iter().
 */
void drm_connector_list_iter_begin(struct drm_device *dev,
                                   struct drm_connector_list_iter *iter)
{
        iter->dev = dev;
        iter->conn = NULL;
        lock_acquire_shared_recursive(&connector_list_iter_dep_map, 0, 1, NULL, _RET_IP_);
}
EXPORT_SYMBOL(drm_connector_list_iter_begin);

/*
 * Extra-safe connector put function that works in any context. Should only be
 * used from the connector_iter functions, where we never really expect to
 * actually release the connector when dropping our final reference.
 */
static void
__drm_connector_put_safe(struct drm_connector *conn)
{
        struct drm_mode_config *config = &conn->dev->mode_config;

        lockdep_assert_held(&config->connector_list_lock);

        if (!refcount_dec_and_test(&conn->base.refcount.refcount))
                return;

        llist_add(&conn->free_node, &config->connector_free_list);
        schedule_work(&config->connector_free_work);
}

/**
 * drm_connector_list_iter_next - return next connector
 * @iter: connector_list iterator
 *
 * Returns: the next connector for @iter, or NULL when the list walk has
 * completed.
 */
struct drm_connector *
drm_connector_list_iter_next(struct drm_connector_list_iter *iter)
{
        struct drm_connector *old_conn = iter->conn;
        struct drm_mode_config *config = &iter->dev->mode_config;
        struct list_head *lhead;
        unsigned long flags;

        spin_lock_irqsave(&config->connector_list_lock, flags);
        lhead = old_conn ? &old_conn->head : &config->connector_list;

        do {
                if (lhead->next == &config->connector_list) {
                        iter->conn = NULL;
                        break;
                }

                lhead = lhead->next;
                iter->conn = list_entry(lhead, struct drm_connector, head);

                /* loop until it's not a zombie connector */
        } while (!kref_get_unless_zero(&iter->conn->base.refcount));

        if (old_conn)
                __drm_connector_put_safe(old_conn);
        spin_unlock_irqrestore(&config->connector_list_lock, flags);

        return iter->conn;
}
EXPORT_SYMBOL(drm_connector_list_iter_next);

/**
 * drm_connector_list_iter_end - tear down a connector_list iterator
 * @iter: connector_list iterator
 *
 * Tears down @iter and releases any resources (like &drm_connector references)
 * acquired while walking the list. This must always be called, both when the
 * iteration completes fully or when it was aborted without walking the entire
 * list.
 */
void drm_connector_list_iter_end(struct drm_connector_list_iter *iter)
{
        struct drm_mode_config *config = &iter->dev->mode_config;
        unsigned long flags;

        iter->dev = NULL;
        if (iter->conn) {
                spin_lock_irqsave(&config->connector_list_lock, flags);
                __drm_connector_put_safe(iter->conn);
                spin_unlock_irqrestore(&config->connector_list_lock, flags);
        }
        lock_release(&connector_list_iter_dep_map, _RET_IP_);
}
EXPORT_SYMBOL(drm_connector_list_iter_end);

static const struct drm_prop_enum_list drm_subpixel_enum_list[] = {
        { SubPixelUnknown, "Unknown" },
        { SubPixelHorizontalRGB, "Horizontal RGB" },
        { SubPixelHorizontalBGR, "Horizontal BGR" },
        { SubPixelVerticalRGB, "Vertical RGB" },
        { SubPixelVerticalBGR, "Vertical BGR" },
        { SubPixelNone, "None" },
};

/**
 * drm_get_subpixel_order_name - return a string for a given subpixel enum
 * @order: enum of subpixel_order
 *
 * Note you could abuse this and return something out of bounds, but that
 * would be a caller error.  No unscrubbed user data should make it here.
 *
 * Returns: string describing an enumerated subpixel property
 */
const char *drm_get_subpixel_order_name(enum subpixel_order order)
{
        return drm_subpixel_enum_list[order].name;
}
EXPORT_SYMBOL(drm_get_subpixel_order_name);

static const struct drm_prop_enum_list drm_dpms_enum_list[] = {
        { DRM_MODE_DPMS_ON, "On" },
        { DRM_MODE_DPMS_STANDBY, "Standby" },
        { DRM_MODE_DPMS_SUSPEND, "Suspend" },
        { DRM_MODE_DPMS_OFF, "Off" }
};
DRM_ENUM_NAME_FN(drm_get_dpms_name, drm_dpms_enum_list)

static const struct drm_prop_enum_list drm_link_status_enum_list[] = {
        { DRM_MODE_LINK_STATUS_GOOD, "Good" },
        { DRM_MODE_LINK_STATUS_BAD, "Bad" },
};

/**
 * drm_display_info_set_bus_formats - set the supported bus formats
 * @info: display info to store bus formats in
 * @formats: array containing the supported bus formats
 * @num_formats: the number of entries in the fmts array
 *
 * Store the supported bus formats in display info structure.
 * See MEDIA_BUS_FMT_* definitions in include/uapi/linux/media-bus-format.h for
 * a full list of available formats.
 *
 * Returns:
 * 0 on success or a negative error code on failure.
 */
int drm_display_info_set_bus_formats(struct drm_display_info *info,
                                     const u32 *formats,
                                     unsigned int num_formats)
{
        u32 *fmts = NULL;

        if (!formats && num_formats)
                return -EINVAL;

        if (formats && num_formats) {
                fmts = kmemdup(formats, sizeof(*formats) * num_formats,
                               GFP_KERNEL);
                if (!fmts)
                        return -ENOMEM;
        }

        kfree(info->bus_formats);
        info->bus_formats = fmts;
        info->num_bus_formats = num_formats;

        return 0;
}
EXPORT_SYMBOL(drm_display_info_set_bus_formats);

/* Optional connector properties. */
static const struct drm_prop_enum_list drm_scaling_mode_enum_list[] = {
        { DRM_MODE_SCALE_NONE, "None" },
        { DRM_MODE_SCALE_FULLSCREEN, "Full" },
        { DRM_MODE_SCALE_CENTER, "Center" },
        { DRM_MODE_SCALE_ASPECT, "Full aspect" },
};

static const struct drm_prop_enum_list drm_aspect_ratio_enum_list[] = {
        { DRM_MODE_PICTURE_ASPECT_NONE, "Automatic" },
        { DRM_MODE_PICTURE_ASPECT_4_3, "4:3" },
        { DRM_MODE_PICTURE_ASPECT_16_9, "16:9" },
};

static const struct drm_prop_enum_list drm_content_type_enum_list[] = {
        { DRM_MODE_CONTENT_TYPE_NO_DATA, "No Data" },
        { DRM_MODE_CONTENT_TYPE_GRAPHICS, "Graphics" },
        { DRM_MODE_CONTENT_TYPE_PHOTO, "Photo" },
        { DRM_MODE_CONTENT_TYPE_CINEMA, "Cinema" },
        { DRM_MODE_CONTENT_TYPE_GAME, "Game" },
};

static const struct drm_prop_enum_list drm_panel_orientation_enum_list[] = {
        { DRM_MODE_PANEL_ORIENTATION_NORMAL,        "Normal"        },
        { DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP,        "Upside Down"        },
        { DRM_MODE_PANEL_ORIENTATION_LEFT_UP,        "Left Side Up"        },
        { DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,        "Right Side Up"        },
};

static const struct drm_prop_enum_list drm_dvi_i_select_enum_list[] = {
        { DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
        { DRM_MODE_SUBCONNECTOR_DVID,      "DVI-D"     }, /* DVI-I  */
        { DRM_MODE_SUBCONNECTOR_DVIA,      "DVI-A"     }, /* DVI-I  */
};
DRM_ENUM_NAME_FN(drm_get_dvi_i_select_name, drm_dvi_i_select_enum_list)

static const struct drm_prop_enum_list drm_dvi_i_subconnector_enum_list[] = {
        { DRM_MODE_SUBCONNECTOR_Unknown,   "Unknown"   }, /* DVI-I, TV-out and DP */
        { DRM_MODE_SUBCONNECTOR_DVID,      "DVI-D"     }, /* DVI-I  */
        { DRM_MODE_SUBCONNECTOR_DVIA,      "DVI-A"     }, /* DVI-I  */
};
DRM_ENUM_NAME_FN(drm_get_dvi_i_subconnector_name,
                 drm_dvi_i_subconnector_enum_list)

static const struct drm_prop_enum_list drm_tv_mode_enum_list[] = {
        { DRM_MODE_TV_MODE_NTSC, "NTSC" },
        { DRM_MODE_TV_MODE_NTSC_443, "NTSC-443" },
        { DRM_MODE_TV_MODE_NTSC_J, "NTSC-J" },
        { DRM_MODE_TV_MODE_PAL, "PAL" },
        { DRM_MODE_TV_MODE_PAL_M, "PAL-M" },
        { DRM_MODE_TV_MODE_PAL_N, "PAL-N" },
        { DRM_MODE_TV_MODE_SECAM, "SECAM" },
        { DRM_MODE_TV_MODE_MONOCHROME, "Mono" },
};
DRM_ENUM_NAME_FN(drm_get_tv_mode_name, drm_tv_mode_enum_list)

/**
 * drm_get_tv_mode_from_name - Translates a TV mode name into its enum value
 * @name: TV Mode name we want to convert
 * @len: Length of @name
 *
 * Translates @name into an enum drm_connector_tv_mode.
 *
 * Returns: the enum value on success, a negative errno otherwise.
 */
int drm_get_tv_mode_from_name(const char *name, size_t len)
{
        unsigned int i;

        for (i = 0; i < ARRAY_SIZE(drm_tv_mode_enum_list); i++) {
                const struct drm_prop_enum_list *item = &drm_tv_mode_enum_list[i];

                if (strlen(item->name) == len && !strncmp(item->name, name, len))
                        return item->type;
        }

        return -EINVAL;
}
EXPORT_SYMBOL(drm_get_tv_mode_from_name);

static const struct drm_prop_enum_list drm_tv_select_enum_list[] = {
        { DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
        { DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
        { DRM_MODE_SUBCONNECTOR_SVIDEO,    "SVIDEO"    }, /* TV-out */
        { DRM_MODE_SUBCONNECTOR_Component, "Component" }, /* TV-out */
        { DRM_MODE_SUBCONNECTOR_SCART,     "SCART"     }, /* TV-out */
};
DRM_ENUM_NAME_FN(drm_get_tv_select_name, drm_tv_select_enum_list)

static const struct drm_prop_enum_list drm_tv_subconnector_enum_list[] = {
        { DRM_MODE_SUBCONNECTOR_Unknown,   "Unknown"   }, /* DVI-I, TV-out and DP */
        { DRM_MODE_SUBCONNECTOR_Composite, "Composite" }, /* TV-out */
        { DRM_MODE_SUBCONNECTOR_SVIDEO,    "SVIDEO"    }, /* TV-out */
        { DRM_MODE_SUBCONNECTOR_Component, "Component" }, /* TV-out */
        { DRM_MODE_SUBCONNECTOR_SCART,     "SCART"     }, /* TV-out */
};
DRM_ENUM_NAME_FN(drm_get_tv_subconnector_name,
                 drm_tv_subconnector_enum_list)

static const struct drm_prop_enum_list drm_dp_subconnector_enum_list[] = {
        { DRM_MODE_SUBCONNECTOR_Unknown,     "Unknown"   }, /* DVI-I, TV-out and DP */
        { DRM_MODE_SUBCONNECTOR_VGA,             "VGA"       }, /* DP */
        { DRM_MODE_SUBCONNECTOR_DVID,             "DVI-D"     }, /* DP */
        { DRM_MODE_SUBCONNECTOR_HDMIA,             "HDMI"      }, /* DP */
        { DRM_MODE_SUBCONNECTOR_DisplayPort, "DP"        }, /* DP */
        { DRM_MODE_SUBCONNECTOR_Wireless,    "Wireless"  }, /* DP */
        { DRM_MODE_SUBCONNECTOR_Native,             "Native"    }, /* DP */
};

DRM_ENUM_NAME_FN(drm_get_dp_subconnector_name,
                 drm_dp_subconnector_enum_list)


static const char * const colorspace_names[] = {
        /* For Default case, driver will set the colorspace */
        [DRM_MODE_COLORIMETRY_DEFAULT] = "Default",
        /* Standard Definition Colorimetry based on CEA 861 */
        [DRM_MODE_COLORIMETRY_SMPTE_170M_YCC] = "SMPTE_170M_YCC",
        [DRM_MODE_COLORIMETRY_BT709_YCC] = "BT709_YCC",
        /* Standard Definition Colorimetry based on IEC 61966-2-4 */
        [DRM_MODE_COLORIMETRY_XVYCC_601] = "XVYCC_601",
        /* High Definition Colorimetry based on IEC 61966-2-4 */
        [DRM_MODE_COLORIMETRY_XVYCC_709] = "XVYCC_709",
        /* Colorimetry based on IEC 61966-2-1/Amendment 1 */
        [DRM_MODE_COLORIMETRY_SYCC_601] = "SYCC_601",
        /* Colorimetry based on IEC 61966-2-5 [33] */
        [DRM_MODE_COLORIMETRY_OPYCC_601] = "opYCC_601",
        /* Colorimetry based on IEC 61966-2-5 */
        [DRM_MODE_COLORIMETRY_OPRGB] = "opRGB",
        /* Colorimetry based on ITU-R BT.2020 */
        [DRM_MODE_COLORIMETRY_BT2020_CYCC] = "BT2020_CYCC",
        /* Colorimetry based on ITU-R BT.2020 */
        [DRM_MODE_COLORIMETRY_BT2020_RGB] = "BT2020_RGB",
        /* Colorimetry based on ITU-R BT.2020 */
        [DRM_MODE_COLORIMETRY_BT2020_YCC] = "BT2020_YCC",
        /* Added as part of Additional Colorimetry Extension in 861.G */
        [DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65] = "DCI-P3_RGB_D65",
        [DRM_MODE_COLORIMETRY_DCI_P3_RGB_THEATER] = "DCI-P3_RGB_Theater",
        [DRM_MODE_COLORIMETRY_RGB_WIDE_FIXED] = "RGB_WIDE_FIXED",
        /* Colorimetry based on scRGB (IEC 61966-2-2) */
        [DRM_MODE_COLORIMETRY_RGB_WIDE_FLOAT] = "RGB_WIDE_FLOAT",
        [DRM_MODE_COLORIMETRY_BT601_YCC] = "BT601_YCC",
};

/**
 * drm_get_colorspace_name - return a string for color encoding
 * @colorspace: color space to compute name of
 *
 * In contrast to the other drm_get_*_name functions this one here returns a
 * const pointer and hence is threadsafe.
 */
const char *drm_get_colorspace_name(enum drm_colorspace colorspace)
{
        if (colorspace < ARRAY_SIZE(colorspace_names) && colorspace_names[colorspace])
                return colorspace_names[colorspace];
        else
                return "(null)";
}

static const u32 hdmi_colorspaces =
        BIT(DRM_MODE_COLORIMETRY_SMPTE_170M_YCC) |
        BIT(DRM_MODE_COLORIMETRY_BT709_YCC) |
        BIT(DRM_MODE_COLORIMETRY_XVYCC_601) |
        BIT(DRM_MODE_COLORIMETRY_XVYCC_709) |
        BIT(DRM_MODE_COLORIMETRY_SYCC_601) |
        BIT(DRM_MODE_COLORIMETRY_OPYCC_601) |
        BIT(DRM_MODE_COLORIMETRY_OPRGB) |
        BIT(DRM_MODE_COLORIMETRY_BT2020_CYCC) |
        BIT(DRM_MODE_COLORIMETRY_BT2020_RGB) |
        BIT(DRM_MODE_COLORIMETRY_BT2020_YCC) |
        BIT(DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65) |
        BIT(DRM_MODE_COLORIMETRY_DCI_P3_RGB_THEATER);

/*
 * As per DP 1.4a spec, 2.2.5.7.5 VSC SDP Payload for Pixel Encoding/Colorimetry
 * Format Table 2-120
 */
static const u32 dp_colorspaces =
        BIT(DRM_MODE_COLORIMETRY_RGB_WIDE_FIXED) |
        BIT(DRM_MODE_COLORIMETRY_RGB_WIDE_FLOAT) |
        BIT(DRM_MODE_COLORIMETRY_OPRGB) |
        BIT(DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65) |
        BIT(DRM_MODE_COLORIMETRY_BT2020_RGB) |
        BIT(DRM_MODE_COLORIMETRY_BT601_YCC) |
        BIT(DRM_MODE_COLORIMETRY_BT709_YCC) |
        BIT(DRM_MODE_COLORIMETRY_XVYCC_601) |
        BIT(DRM_MODE_COLORIMETRY_XVYCC_709) |
        BIT(DRM_MODE_COLORIMETRY_SYCC_601) |
        BIT(DRM_MODE_COLORIMETRY_OPYCC_601) |
        BIT(DRM_MODE_COLORIMETRY_BT2020_CYCC) |
        BIT(DRM_MODE_COLORIMETRY_BT2020_YCC);

static const struct drm_prop_enum_list broadcast_rgb_names[] = {
        { DRM_HDMI_BROADCAST_RGB_AUTO, "Automatic" },
        { DRM_HDMI_BROADCAST_RGB_FULL, "Full" },
        { DRM_HDMI_BROADCAST_RGB_LIMITED, "Limited 16:235" },
};

/*
 * drm_hdmi_connector_get_broadcast_rgb_name - Return a string for HDMI connector RGB broadcast selection
 * @broadcast_rgb: Broadcast RGB selection to compute name of
 *
 * Returns: the name of the Broadcast RGB selection, or NULL if the type
 * is not valid.
 */
const char *
drm_hdmi_connector_get_broadcast_rgb_name(enum drm_hdmi_broadcast_rgb broadcast_rgb)
{
        if (broadcast_rgb >= ARRAY_SIZE(broadcast_rgb_names))
                return NULL;

        return broadcast_rgb_names[broadcast_rgb].name;
}
EXPORT_SYMBOL(drm_hdmi_connector_get_broadcast_rgb_name);

static const char * const output_format_str[] = {
        [HDMI_COLORSPACE_RGB]                = "RGB",
        [HDMI_COLORSPACE_YUV420]        = "YUV 4:2:0",
        [HDMI_COLORSPACE_YUV422]        = "YUV 4:2:2",
        [HDMI_COLORSPACE_YUV444]        = "YUV 4:4:4",
};

/*
 * drm_hdmi_connector_get_output_format_name() - Return a string for HDMI connector output format
 * @fmt: Output format to compute name of
 *
 * Returns: the name of the output format, or NULL if the type is not
 * valid.
 */
const char *
drm_hdmi_connector_get_output_format_name(enum hdmi_colorspace fmt)
{
        if (fmt >= ARRAY_SIZE(output_format_str))
                return NULL;

        return output_format_str[fmt];
}
EXPORT_SYMBOL(drm_hdmi_connector_get_output_format_name);

/**
 * DOC: standard connector properties
 *
 * DRM connectors have a few standardized properties:
 *
 * EDID:
 *         Blob property which contains the current EDID read from the sink. This
 *         is useful to parse sink identification information like vendor, model
 *         and serial. Drivers should update this property by calling
 *         drm_connector_update_edid_property(), usually after having parsed
 *         the EDID using drm_add_edid_modes(). Userspace cannot change this
 *         property.
 *
 *         User-space should not parse the EDID to obtain information exposed via
 *         other KMS properties (because the kernel might apply limits, quirks or
 *         fixups to the EDID). For instance, user-space should not try to parse
 *         mode lists from the EDID.
 * DPMS:
 *         Legacy property for setting the power state of the connector. For atomic
 *         drivers this is only provided for backwards compatibility with existing
 *         drivers, it remaps to controlling the "ACTIVE" property on the CRTC the
 *         connector is linked to. Drivers should never set this property directly,
 *         it is handled by the DRM core by calling the &drm_connector_funcs.dpms
 *         callback. For atomic drivers the remapping to the "ACTIVE" property is
 *         implemented in the DRM core.
 *
 *         Note that this property cannot be set through the MODE_ATOMIC ioctl,
 *         userspace must use "ACTIVE" on the CRTC instead.
 *
 *         WARNING:
 *
 *         For userspace also running on legacy drivers the "DPMS" semantics are a
 *         lot more complicated. First, userspace cannot rely on the "DPMS" value
 *         returned by the GETCONNECTOR actually reflecting reality, because many
 *         drivers fail to update it. For atomic drivers this is taken care of in
 *         drm_atomic_helper_update_legacy_modeset_state().
 *
 *         The second issue is that the DPMS state is only well-defined when the
 *         connector is connected to a CRTC. In atomic the DRM core enforces that
 *         "ACTIVE" is off in such a case, no such checks exists for "DPMS".
 *
 *         Finally, when enabling an output using the legacy SETCONFIG ioctl then
 *         "DPMS" is forced to ON. But see above, that might not be reflected in
 *         the software value on legacy drivers.
 *
 *         Summarizing: Only set "DPMS" when the connector is known to be enabled,
 *         assume that a successful SETCONFIG call also sets "DPMS" to on, and
 *         never read back the value of "DPMS" because it can be incorrect.
 * PATH:
 *         Connector path property to identify how this sink is physically
 *         connected. Used by DP MST. This should be set by calling
 *         drm_connector_set_path_property(), in the case of DP MST with the
 *         path property the MST manager created. Userspace cannot change this
 *         property.
 *
 *         In the case of DP MST, the property has the format
 *         ``mst:<parent>-<ports>`` where ``<parent>`` is the KMS object ID of the
 *         parent connector and ``<ports>`` is a hyphen-separated list of DP MST
 *         port numbers. Note, KMS object IDs are not guaranteed to be stable
 *         across reboots.
 * TILE:
 *         Connector tile group property to indicate how a set of DRM connector
 *         compose together into one logical screen. This is used by both high-res
 *         external screens (often only using a single cable, but exposing multiple
 *         DP MST sinks), or high-res integrated panels (like dual-link DSI) which
 *         are not gen-locked. Note that for tiled panels which are genlocked, like
 *         dual-link LVDS or dual-link DSI, the driver should try to not expose the
 *         tiling and virtualise both &drm_crtc and &drm_plane if needed. Drivers
 *         should update this value using drm_connector_set_tile_property().
 *         Userspace cannot change this property.
 * link-status:
 *      Connector link-status property to indicate the status of link. The
 *      default value of link-status is "GOOD". If something fails during or
 *      after modeset, the kernel driver may set this to "BAD" and issue a
 *      hotplug uevent. Drivers should update this value using
 *      drm_connector_set_link_status_property().
 *
 *      When user-space receives the hotplug uevent and detects a "BAD"
 *      link-status, the sink doesn't receive pixels anymore (e.g. the screen
 *      becomes completely black). The list of available modes may have
 *      changed. User-space is expected to pick a new mode if the current one
 *      has disappeared and perform a new modeset with link-status set to
 *      "GOOD" to re-enable the connector.
 *
 *      If multiple connectors share the same CRTC and one of them gets a "BAD"
 *      link-status, the other are unaffected (ie. the sinks still continue to
 *      receive pixels).
 *
 *      When user-space performs an atomic commit on a connector with a "BAD"
 *      link-status without resetting the property to "GOOD", the sink may
 *      still not receive pixels. When user-space performs an atomic commit
 *      which resets the link-status property to "GOOD" without the
 *      ALLOW_MODESET flag set, it might fail because a modeset is required.
 *
 *      User-space can only change link-status to "GOOD", changing it to "BAD"
 *      is a no-op.
 *
 *      For backwards compatibility with non-atomic userspace the kernel
 *      tries to automatically set the link-status back to "GOOD" in the
 *      SETCRTC IOCTL. This might fail if the mode is no longer valid, similar
 *      to how it might fail if a different screen has been connected in the
 *      interim.
 * non_desktop:
 *         Indicates the output should be ignored for purposes of displaying a
 *         standard desktop environment or console. This is most likely because
 *         the output device is not rectilinear.
 * Content Protection:
 *        This property is used by userspace to request the kernel protect future
 *        content communicated over the link. When requested, kernel will apply
 *        the appropriate means of protection (most often HDCP), and use the
 *        property to tell userspace the protection is active.
 *
 *        Drivers can set this up by calling
 *        drm_connector_attach_content_protection_property() on initialization.
 *
 *        The value of this property can be one of the following:
 *
 *        DRM_MODE_CONTENT_PROTECTION_UNDESIRED = 0
 *                The link is not protected, content is transmitted in the clear.
 *        DRM_MODE_CONTENT_PROTECTION_DESIRED = 1
 *                Userspace has requested content protection, but the link is not
 *                currently protected. When in this state, kernel should enable
 *                Content Protection as soon as possible.
 *        DRM_MODE_CONTENT_PROTECTION_ENABLED = 2
 *                Userspace has requested content protection, and the link is
 *                protected. Only the driver can set the property to this value.
 *                If userspace attempts to set to ENABLED, kernel will return
 *                -EINVAL.
 *
 *        A few guidelines:
 *
 *        - DESIRED state should be preserved until userspace de-asserts it by
 *          setting the property to UNDESIRED. This means ENABLED should only
 *          transition to UNDESIRED when the user explicitly requests it.
 *        - If the state is DESIRED, kernel should attempt to re-authenticate the
 *          link whenever possible. This includes across disable/enable, dpms,
 *          hotplug, downstream device changes, link status failures, etc..
 *        - Kernel sends uevent with the connector id and property id through
 *          @drm_hdcp_update_content_protection, upon below kernel triggered
 *          scenarios:
 *
 *                - DESIRED -> ENABLED (authentication success)
 *                - ENABLED -> DESIRED (termination of authentication)
 *        - Please note no uevents for userspace triggered property state changes,
 *          which can't fail such as
 *
 *                - DESIRED/ENABLED -> UNDESIRED
 *                - UNDESIRED -> DESIRED
 *        - Userspace is responsible for polling the property or listen to uevents
 *          to determine when the value transitions from ENABLED to DESIRED.
 *          This signifies the link is no longer protected and userspace should
 *          take appropriate action (whatever that might be).
 *
 * HDCP Content Type:
 *        This Enum property is used by the userspace to declare the content type
 *        of the display stream, to kernel. Here display stream stands for any
 *        display content that userspace intended to display through HDCP
 *        encryption.
 *
 *        Content Type of a stream is decided by the owner of the stream, as
 *        "HDCP Type0" or "HDCP Type1".
 *
 *        The value of the property can be one of the below:
 *          - "HDCP Type0": DRM_MODE_HDCP_CONTENT_TYPE0 = 0
 *          - "HDCP Type1": DRM_MODE_HDCP_CONTENT_TYPE1 = 1
 *
 *        When kernel starts the HDCP authentication (see "Content Protection"
 *        for details), it uses the content type in "HDCP Content Type"
 *        for performing the HDCP authentication with the display sink.
 *
 *        Please note in HDCP spec versions, a link can be authenticated with
 *        HDCP 2.2 for Content Type 0/Content Type 1. Where as a link can be
 *        authenticated with HDCP1.4 only for Content Type 0(though it is implicit
 *        in nature. As there is no reference for Content Type in HDCP1.4).
 *
 *        HDCP2.2 authentication protocol itself takes the "Content Type" as a
 *        parameter, which is a input for the DP HDCP2.2 encryption algo.
 *
 *        In case of Type 0 content protection request, kernel driver can choose
 *        either of HDCP spec versions 1.4 and 2.2. When HDCP2.2 is used for
 *        "HDCP Type 0", a HDCP 2.2 capable repeater in the downstream can send
 *        that content to a HDCP 1.4 authenticated HDCP sink (Type0 link).
 *        But if the content is classified as "HDCP Type 1", above mentioned
 *        HDCP 2.2 repeater wont send the content to the HDCP sink as it can't
 *        authenticate the HDCP1.4 capable sink for "HDCP Type 1".
 *
 *        Please note userspace can be ignorant of the HDCP versions used by the
 *        kernel driver to achieve the "HDCP Content Type".
 *
 *        At current scenario, classifying a content as Type 1 ensures that the
 *        content will be displayed only through the HDCP2.2 encrypted link.
 *
 *        Note that the HDCP Content Type property is introduced at HDCP 2.2, and
 *        defaults to type 0. It is only exposed by drivers supporting HDCP 2.2
 *        (hence supporting Type 0 and Type 1). Based on how next versions of
 *        HDCP specs are defined content Type could be used for higher versions
 *        too.
 *
 *        If content type is changed when "Content Protection" is not UNDESIRED,
 *        then kernel will disable the HDCP and re-enable with new type in the
 *        same atomic commit. And when "Content Protection" is ENABLED, it means
 *        that link is HDCP authenticated and encrypted, for the transmission of
 *        the Type of stream mentioned at "HDCP Content Type".
 *
 * HDR_OUTPUT_METADATA:
 *        Connector property to enable userspace to send HDR Metadata to
 *        driver. This metadata is based on the composition and blending
 *        policies decided by user, taking into account the hardware and
 *        sink capabilities. The driver gets this metadata and creates a
 *        Dynamic Range and Mastering Infoframe (DRM) in case of HDMI,
 *        SDP packet (Non-audio INFOFRAME SDP v1.3) for DP. This is then
 *        sent to sink. This notifies the sink of the upcoming frame's Color
 *        Encoding and Luminance parameters.
 *
 *        Userspace first need to detect the HDR capabilities of sink by
 *        reading and parsing the EDID. Details of HDR metadata for HDMI
 *        are added in CTA 861.G spec. For DP , its defined in VESA DP
 *        Standard v1.4. It needs to then get the metadata information
 *        of the video/game/app content which are encoded in HDR (basically
 *        using HDR transfer functions). With this information it needs to
 *        decide on a blending policy and compose the relevant
 *        layers/overlays into a common format. Once this blending is done,
 *        userspace will be aware of the metadata of the composed frame to
 *        be send to sink. It then uses this property to communicate this
 *        metadata to driver which then make a Infoframe packet and sends
 *        to sink based on the type of encoder connected.
 *
 *        Userspace will be responsible to do Tone mapping operation in case:
 *                - Some layers are HDR and others are SDR
 *                - HDR layers luminance is not same as sink
 *
 *        It will even need to do colorspace conversion and get all layers
 *        to one common colorspace for blending. It can use either GL, Media
 *        or display engine to get this done based on the capabilities of the
 *        associated hardware.
 *
 *        Driver expects metadata to be put in &struct hdr_output_metadata
 *        structure from userspace. This is received as blob and stored in
 *        &drm_connector_state.hdr_output_metadata. It parses EDID and saves the
 *        sink metadata in &struct hdr_sink_metadata, as
 *        &drm_connector.hdr_sink_metadata.  Driver uses
 *        drm_hdmi_infoframe_set_hdr_metadata() helper to set the HDR metadata,
 *        hdmi_drm_infoframe_pack() to pack the infoframe as per spec, in case of
 *        HDMI encoder.
 *
 * max bpc:
 *        This range property is used by userspace to limit the bit depth. When
 *        used the driver would limit the bpc in accordance with the valid range
 *        supported by the hardware and sink. Drivers to use the function
 *        drm_connector_attach_max_bpc_property() to create and attach the
 *        property to the connector during initialization.
 *
 * Connectors also have one standardized atomic property:
 *
 * CRTC_ID:
 *         Mode object ID of the &drm_crtc this connector should be connected to.
 *
 * Connectors for LCD panels may also have one standardized property:
 *
 * panel orientation:
 *        On some devices the LCD panel is mounted in the casing in such a way
 *        that the up/top side of the panel does not match with the top side of
 *        the device. Userspace can use this property to check for this.
 *        Note that input coordinates from touchscreens (input devices with
 *        INPUT_PROP_DIRECT) will still map 1:1 to the actual LCD panel
 *        coordinates, so if userspace rotates the picture to adjust for
 *        the orientation it must also apply the same transformation to the
 *        touchscreen input coordinates. This property is initialized by calling
 *        drm_connector_set_panel_orientation() or
 *        drm_connector_set_panel_orientation_with_quirk()
 *
 * scaling mode:
 *        This property defines how a non-native mode is upscaled to the native
 *        mode of an LCD panel:
 *
 *        None:
 *                No upscaling happens, scaling is left to the panel. Not all
 *                drivers expose this mode.
 *        Full:
 *                The output is upscaled to the full resolution of the panel,
 *                ignoring the aspect ratio.
 *        Center:
 *                No upscaling happens, the output is centered within the native
 *                resolution the panel.
 *        Full aspect:
 *                The output is upscaled to maximize either the width or height
 *                while retaining the aspect ratio.
 *
 *        This property should be set up by calling
 *        drm_connector_attach_scaling_mode_property(). Note that drivers
 *        can also expose this property to external outputs, in which case they
 *        must support "None", which should be the default (since external screens
 *        have a built-in scaler).
 *
 * subconnector:
 *        This property is used by DVI-I, TVout and DisplayPort to indicate different
 *        connector subtypes. Enum values more or less match with those from main
 *        connector types.
 *        For DVI-I and TVout there is also a matching property "select subconnector"
 *        allowing to switch between signal types.
 *        DP subconnector corresponds to a downstream port.
 *
 * privacy-screen sw-state, privacy-screen hw-state:
 *        These 2 optional properties can be used to query the state of the
 *        electronic privacy screen that is available on some displays; and in
 *        some cases also control the state. If a driver implements these
 *        properties then both properties must be present.
 *
 *        "privacy-screen hw-state" is read-only and reflects the actual state
 *        of the privacy-screen, possible values: "Enabled", "Disabled,
 *        "Enabled-locked", "Disabled-locked". The locked states indicate
 *        that the state cannot be changed through the DRM API. E.g. there
 *        might be devices where the firmware-setup options, or a hardware
 *        slider-switch, offer always on / off modes.
 *
 *        "privacy-screen sw-state" can be set to change the privacy-screen state
 *        when not locked. In this case the driver must update the hw-state
 *        property to reflect the new state on completion of the commit of the
 *        sw-state property. Setting the sw-state property when the hw-state is
 *        locked must be interpreted by the driver as a request to change the
 *        state to the set state when the hw-state becomes unlocked. E.g. if
 *        "privacy-screen hw-state" is "Enabled-locked" and the sw-state
 *        gets set to "Disabled" followed by the user unlocking the state by
 *        changing the slider-switch position, then the driver must set the
 *        state to "Disabled" upon receiving the unlock event.
 *
 *        In some cases the privacy-screen's actual state might change outside of
 *        control of the DRM code. E.g. there might be a firmware handled hotkey
 *        which toggles the actual state, or the actual state might be changed
 *        through another userspace API such as writing /proc/acpi/ibm/lcdshadow.
 *        In this case the driver must update both the hw-state and the sw-state
 *        to reflect the new value, overwriting any pending state requests in the
 *        sw-state. Any pending sw-state requests are thus discarded.
 *
 *        Note that the ability for the state to change outside of control of
 *        the DRM master process means that userspace must not cache the value
 *        of the sw-state. Caching the sw-state value and including it in later
 *        atomic commits may lead to overriding a state change done through e.g.
 *        a firmware handled hotkey. Therefor userspace must not include the
 *        privacy-screen sw-state in an atomic commit unless it wants to change
 *        its value.
 *
 * left margin, right margin, top margin, bottom margin:
 *        Add margins to the connector's viewport. This is typically used to
 *        mitigate overscan on TVs.
 *
 *        The value is the size in pixels of the black border which will be
 *        added. The attached CRTC's content will be scaled to fill the whole
 *        area inside the margin.
 *
 *        The margins configuration might be sent to the sink, e.g. via HDMI AVI
 *        InfoFrames.
 *
 *        Drivers can set up these properties by calling
 *        drm_mode_create_tv_margin_properties().
 */

int drm_connector_create_standard_properties(struct drm_device *dev)
{
        struct drm_property *prop;

        prop = drm_property_create(dev, DRM_MODE_PROP_BLOB |
                                   DRM_MODE_PROP_IMMUTABLE,
                                   "EDID", 0);
        if (!prop)
                return -ENOMEM;
        dev->mode_config.edid_property = prop;

        prop = drm_property_create_enum(dev, 0,
                                   "DPMS", drm_dpms_enum_list,
                                   ARRAY_SIZE(drm_dpms_enum_list));
        if (!prop)
                return -ENOMEM;
        dev->mode_config.dpms_property = prop;

        prop = drm_property_create(dev,
                                   DRM_MODE_PROP_BLOB |
                                   DRM_MODE_PROP_IMMUTABLE,
                                   "PATH", 0);
        if (!prop)
                return -ENOMEM;
        dev->mode_config.path_property = prop;

        prop = drm_property_create(dev,
                                   DRM_MODE_PROP_BLOB |
                                   DRM_MODE_PROP_IMMUTABLE,
                                   "TILE", 0);
        if (!prop)
                return -ENOMEM;
        dev->mode_config.tile_property = prop;

        prop = drm_property_create_enum(dev, 0, "link-status",
                                        drm_link_status_enum_list,
                                        ARRAY_SIZE(drm_link_status_enum_list));
        if (!prop)
                return -ENOMEM;
        dev->mode_config.link_status_property = prop;

        prop = drm_property_create_bool(dev, DRM_MODE_PROP_IMMUTABLE, "non-desktop");
        if (!prop)
                return -ENOMEM;
        dev->mode_config.non_desktop_property = prop;

        prop = drm_property_create(dev, DRM_MODE_PROP_BLOB,
                                   "HDR_OUTPUT_METADATA", 0);
        if (!prop)
                return -ENOMEM;
        dev->mode_config.hdr_output_metadata_property = prop;

        return 0;
}

/**
 * drm_mode_create_dvi_i_properties - create DVI-I specific connector properties
 * @dev: DRM device
 *
 * Called by a driver the first time a DVI-I connector is made.
 *
 * Returns: %0
 */
int drm_mode_create_dvi_i_properties(struct drm_device *dev)
{
        struct drm_property *dvi_i_selector;
        struct drm_property *dvi_i_subconnector;

        if (dev->mode_config.dvi_i_select_subconnector_property)
                return 0;

        dvi_i_selector =
                drm_property_create_enum(dev, 0,
                                    "select subconnector",
                                    drm_dvi_i_select_enum_list,
                                    ARRAY_SIZE(drm_dvi_i_select_enum_list));
        dev->mode_config.dvi_i_select_subconnector_property = dvi_i_selector;

        dvi_i_subconnector = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
                                    "subconnector",
                                    drm_dvi_i_subconnector_enum_list,
                                    ARRAY_SIZE(drm_dvi_i_subconnector_enum_list));
        dev->mode_config.dvi_i_subconnector_property = dvi_i_subconnector;

        return 0;
}
EXPORT_SYMBOL(drm_mode_create_dvi_i_properties);

/**
 * drm_connector_attach_dp_subconnector_property - create subconnector property for DP
 * @connector: drm_connector to attach property
 *
 * Called by a driver when DP connector is created.
 */
void drm_connector_attach_dp_subconnector_property(struct drm_connector *connector)
{
        struct drm_mode_config *mode_config = &connector->dev->mode_config;

        if (!mode_config->dp_subconnector_property)
                mode_config->dp_subconnector_property =
                        drm_property_create_enum(connector->dev,
                                DRM_MODE_PROP_IMMUTABLE,
                                "subconnector",
                                drm_dp_subconnector_enum_list,
                                ARRAY_SIZE(drm_dp_subconnector_enum_list));

        drm_object_attach_property(&connector->base,
                                   mode_config->dp_subconnector_property,
                                   DRM_MODE_SUBCONNECTOR_Unknown);
}
EXPORT_SYMBOL(drm_connector_attach_dp_subconnector_property);

/**
 * DOC: HDMI connector properties
 *
 * Broadcast RGB (HDMI specific)
 *      Indicates the Quantization Range (Full vs Limited) used. The color
 *      processing pipeline will be adjusted to match the value of the
 *      property, and the Infoframes will be generated and sent accordingly.
 *
 *      This property is only relevant if the HDMI output format is RGB. If
 *      it's one of the YCbCr variant, it will be ignored.
 *
 *      The CRTC attached to the connector must be configured by user-space to
 *      always produce full-range pixels.
 *
 *      The value of this property can be one of the following:
 *
 *      Automatic:
 *              The quantization range is selected automatically based on the
 *              mode according to the HDMI specifications (HDMI 1.4b - Section
 *              6.6 - Video Quantization Ranges).
 *
 *      Full:
 *              Full quantization range is forced.
 *
 *      Limited 16:235:
 *              Limited quantization range is forced. Unlike the name suggests,
 *              this works for any number of bits-per-component.
 *
 *      Property values other than Automatic can result in colors being off (if
 *      limited is selected but the display expects full), or a black screen
 *      (if full is selected but the display expects limited).
 *
 *      Drivers can set up this property by calling
 *      drm_connector_attach_broadcast_rgb_property().
 *
 * content type (HDMI specific):
 *        Indicates content type setting to be used in HDMI infoframes to indicate
 *        content type for the external device, so that it adjusts its display
 *        settings accordingly.
 *
 *        The value of this property can be one of the following:
 *
 *        No Data:
 *                Content type is unknown
 *        Graphics:
 *                Content type is graphics
 *        Photo:
 *                Content type is photo
 *        Cinema:
 *                Content type is cinema
 *        Game:
 *                Content type is game
 *
 *        The meaning of each content type is defined in CTA-861-G table 15.
 *
 *        Drivers can set up this property by calling
 *        drm_connector_attach_content_type_property(). Decoding to
 *        infoframe values is done through drm_hdmi_avi_infoframe_content_type().
 */

/*
 * TODO: Document the properties:
 *   - brightness
 *   - contrast
 *   - flicker reduction
 *   - hue
 *   - mode
 *   - overscan
 *   - saturation
 *   - select subconnector
 */
/**
 * DOC: Analog TV Connector Properties
 *
 * TV Mode:
 *        Indicates the TV Mode used on an analog TV connector. The value
 *        of this property can be one of the following:
 *
 *        NTSC:
 *                TV Mode is CCIR System M (aka 525-lines) together with
 *                the NTSC Color Encoding.
 *
 *        NTSC-443:
 *
 *                TV Mode is CCIR System M (aka 525-lines) together with
 *                the NTSC Color Encoding, but with a color subcarrier
 *                frequency of 4.43MHz
 *
 *        NTSC-J:
 *
 *                TV Mode is CCIR System M (aka 525-lines) together with
 *                the NTSC Color Encoding, but with a black level equal to
 *                the blanking level.
 *
 *        PAL:
 *
 *                TV Mode is CCIR System B (aka 625-lines) together with
 *                the PAL Color Encoding.
 *
 *        PAL-M:
 *
 *                TV Mode is CCIR System M (aka 525-lines) together with
 *                the PAL Color Encoding.
 *
 *        PAL-N:
 *
 *                TV Mode is CCIR System N together with the PAL Color
 *                Encoding, a color subcarrier frequency of 3.58MHz, the
 *                SECAM color space, and narrower channels than other PAL
 *                variants.
 *
 *        SECAM:
 *
 *                TV Mode is CCIR System B (aka 625-lines) together with
 *                the SECAM Color Encoding.
 *
 *        Mono:
 *
 *                Use timings appropriate to the DRM mode, including
 *                equalizing pulses for a 525-line or 625-line mode,
 *                with no pedestal or color encoding.
 *
 *        Drivers can set up this property by calling
 *        drm_mode_create_tv_properties().
 */

/**
 * drm_connector_attach_content_type_property - attach content-type property
 * @connector: connector to attach content type property on.
 *
 * Called by a driver the first time a HDMI connector is made.
 *
 * Returns: %0
 */
int drm_connector_attach_content_type_property(struct drm_connector *connector)
{
        if (!drm_mode_create_content_type_property(connector->dev))
                drm_object_attach_property(&connector->base,
                                           connector->dev->mode_config.content_type_property,
                                           DRM_MODE_CONTENT_TYPE_NO_DATA);
        return 0;
}
EXPORT_SYMBOL(drm_connector_attach_content_type_property);

/**
 * drm_connector_attach_tv_margin_properties - attach TV connector margin
 *         properties
 * @connector: DRM connector
 *
 * Called by a driver when it needs to attach TV margin props to a connector.
 * Typically used on SDTV and HDMI connectors.
 */
void drm_connector_attach_tv_margin_properties(struct drm_connector *connector)
{
        struct drm_device *dev = connector->dev;

        drm_object_attach_property(&connector->base,
                                   dev->mode_config.tv_left_margin_property,
                                   0);
        drm_object_attach_property(&connector->base,
                                   dev->mode_config.tv_right_margin_property,
                                   0);
        drm_object_attach_property(&connector->base,
                                   dev->mode_config.tv_top_margin_property,
                                   0);
        drm_object_attach_property(&connector->base,
                                   dev->mode_config.tv_bottom_margin_property,
                                   0);
}
EXPORT_SYMBOL(drm_connector_attach_tv_margin_properties);

/**
 * drm_mode_create_tv_margin_properties - create TV connector margin properties
 * @dev: DRM device
 *
 * Called by a driver's HDMI connector initialization routine, this function
 * creates the TV margin properties for a given device. No need to call this
 * function for an SDTV connector, it's already called from
 * drm_mode_create_tv_properties_legacy().
 *
 * Returns:
 * 0 on success or a negative error code on failure.
 */
int drm_mode_create_tv_margin_properties(struct drm_device *dev)
{
        if (dev->mode_config.tv_left_margin_property)
                return 0;

        dev->mode_config.tv_left_margin_property =
                drm_property_create_range(dev, 0, "left margin", 0, 100);
        if (!dev->mode_config.tv_left_margin_property)
                return -ENOMEM;

        dev->mode_config.tv_right_margin_property =
                drm_property_create_range(dev, 0, "right margin", 0, 100);
        if (!dev->mode_config.tv_right_margin_property)
                return -ENOMEM;

        dev->mode_config.tv_top_margin_property =
                drm_property_create_range(dev, 0, "top margin", 0, 100);
        if (!dev->mode_config.tv_top_margin_property)
                return -ENOMEM;

        dev->mode_config.tv_bottom_margin_property =
                drm_property_create_range(dev, 0, "bottom margin", 0, 100);
        if (!dev->mode_config.tv_bottom_margin_property)
                return -ENOMEM;

        return 0;
}
EXPORT_SYMBOL(drm_mode_create_tv_margin_properties);

/**
 * drm_mode_create_tv_properties_legacy - create TV specific connector properties
 * @dev: DRM device
 * @num_modes: number of different TV formats (modes) supported
 * @modes: array of pointers to strings containing name of each format
 *
 * Called by a driver's TV initialization routine, this function creates
 * the TV specific connector properties for a given device.  Caller is
 * responsible for allocating a list of format names and passing them to
 * this routine.
 *
 * NOTE: This functions registers the deprecated "mode" connector
 * property to select the analog TV mode (ie, NTSC, PAL, etc.). New
 * drivers must use drm_mode_create_tv_properties() instead.
 *
 * Returns:
 * 0 on success or a negative error code on failure.
 */
int drm_mode_create_tv_properties_legacy(struct drm_device *dev,
                                         unsigned int num_modes,
                                         const char * const modes[])
{
        struct drm_property *tv_selector;
        struct drm_property *tv_subconnector;
        unsigned int i;

        if (dev->mode_config.tv_select_subconnector_property)
                return 0;

        /*
         * Basic connector properties
         */
        tv_selector = drm_property_create_enum(dev, 0,
                                          "select subconnector",
                                          drm_tv_select_enum_list,
                                          ARRAY_SIZE(drm_tv_select_enum_list));
        if (!tv_selector)
                goto nomem;

        dev->mode_config.tv_select_subconnector_property = tv_selector;

        tv_subconnector =
                drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
                                    "subconnector",
                                    drm_tv_subconnector_enum_list,
                                    ARRAY_SIZE(drm_tv_subconnector_enum_list));
        if (!tv_subconnector)
                goto nomem;
        dev->mode_config.tv_subconnector_property = tv_subconnector;

        /*
         * Other, TV specific properties: margins & TV modes.
         */
        if (drm_mode_create_tv_margin_properties(dev))
                goto nomem;

        if (num_modes) {
                dev->mode_config.legacy_tv_mode_property =
                        drm_property_create(dev, DRM_MODE_PROP_ENUM,
                                            "mode", num_modes);
                if (!dev->mode_config.legacy_tv_mode_property)
                        goto nomem;

                for (i = 0; i < num_modes; i++)
                        drm_property_add_enum(dev->mode_config.legacy_tv_mode_property,
                                              i, modes[i]);
        }

        dev->mode_config.tv_brightness_property =
                drm_property_create_range(dev, 0, "brightness", 0, 100);
        if (!dev->mode_config.tv_brightness_property)
                goto nomem;

        dev->mode_config.tv_contrast_property =
                drm_property_create_range(dev, 0, "contrast", 0, 100);
        if (!dev->mode_config.tv_contrast_property)
                goto nomem;

        dev->mode_config.tv_flicker_reduction_property =
                drm_property_create_range(dev, 0, "flicker reduction", 0, 100);
        if (!dev->mode_config.tv_flicker_reduction_property)
                goto nomem;

        dev->mode_config.tv_overscan_property =
                drm_property_create_range(dev, 0, "overscan", 0, 100);
        if (!dev->mode_config.tv_overscan_property)
                goto nomem;

        dev->mode_config.tv_saturation_property =
                drm_property_create_range(dev, 0, "saturation", 0, 100);
        if (!dev->mode_config.tv_saturation_property)
                goto nomem;

        dev->mode_config.tv_hue_property =
                drm_property_create_range(dev, 0, "hue", 0, 100);
        if (!dev->mode_config.tv_hue_property)
                goto nomem;

        return 0;
nomem:
        return -ENOMEM;
}
EXPORT_SYMBOL(drm_mode_create_tv_properties_legacy);

/**
 * drm_mode_create_tv_properties - create TV specific connector properties
 * @dev: DRM device
 * @supported_tv_modes: Bitmask of TV modes supported (See DRM_MODE_TV_MODE_*)
 *
 * Called by a driver's TV initialization routine, this function creates
 * the TV specific connector properties for a given device.
 *
 * Returns:
 * 0 on success or a negative error code on failure.
 */
int drm_mode_create_tv_properties(struct drm_device *dev,
                                  unsigned int supported_tv_modes)
{
        struct drm_prop_enum_list tv_mode_list[DRM_MODE_TV_MODE_MAX];
        struct drm_property *tv_mode;
        unsigned int i, len = 0;

        if (dev->mode_config.tv_mode_property)
                return 0;

        for (i = 0; i < DRM_MODE_TV_MODE_MAX; i++) {
                if (!(supported_tv_modes & BIT(i)))
                        continue;

                tv_mode_list[len].type = i;
                tv_mode_list[len].name = drm_get_tv_mode_name(i);
                len++;
        }

        tv_mode = drm_property_create_enum(dev, 0, "TV mode",
                                           tv_mode_list, len);
        if (!tv_mode)
                return -ENOMEM;

        dev->mode_config.tv_mode_property = tv_mode;

        return drm_mode_create_tv_properties_legacy(dev, 0, NULL);
}
EXPORT_SYMBOL(drm_mode_create_tv_properties);

/**
 * drm_mode_create_scaling_mode_property - create scaling mode property
 * @dev: DRM device
 *
 * Called by a driver the first time it's needed, must be attached to desired
 * connectors.
 *
 * Atomic drivers should use drm_connector_attach_scaling_mode_property()
 * instead to correctly assign &drm_connector_state.scaling_mode
 * in the atomic state.
 *
 * Returns: %0
 */
int drm_mode_create_scaling_mode_property(struct drm_device *dev)
{
        struct drm_property *scaling_mode;

        if (dev->mode_config.scaling_mode_property)
                return 0;

        scaling_mode =
                drm_property_create_enum(dev, 0, "scaling mode",
                                drm_scaling_mode_enum_list,
                                    ARRAY_SIZE(drm_scaling_mode_enum_list));

        dev->mode_config.scaling_mode_property = scaling_mode;

        return 0;
}
EXPORT_SYMBOL(drm_mode_create_scaling_mode_property);

/**
 * DOC: Variable refresh properties
 *
 * Variable refresh rate capable displays can dynamically adjust their
 * refresh rate by extending the duration of their vertical front porch
 * until page flip or timeout occurs. This can reduce or remove stuttering
 * and latency in scenarios where the page flip does not align with the
 * vblank interval.
 *
 * An example scenario would be an application flipping at a constant rate
 * of 48Hz on a 60Hz display. The page flip will frequently miss the vblank
 * interval and the same contents will be displayed twice. This can be
 * observed as stuttering for content with motion.
 *
 * If variable refresh rate was active on a display that supported a
 * variable refresh range from 35Hz to 60Hz no stuttering would be observable
 * for the example scenario. The minimum supported variable refresh rate of
 * 35Hz is below the page flip frequency and the vertical front porch can
 * be extended until the page flip occurs. The vblank interval will be
 * directly aligned to the page flip rate.
 *
 * Not all userspace content is suitable for use with variable refresh rate.
 * Large and frequent changes in vertical front porch duration may worsen
 * perceived stuttering for input sensitive applications.
 *
 * Panel brightness will also vary with vertical front porch duration. Some
 * panels may have noticeable differences in brightness between the minimum
 * vertical front porch duration and the maximum vertical front porch duration.
 * Large and frequent changes in vertical front porch duration may produce
 * observable flickering for such panels.
 *
 * Userspace control for variable refresh rate is supported via properties
 * on the &drm_connector and &drm_crtc objects.
 *
 * "vrr_capable":
 *        Optional &drm_connector boolean property that drivers should attach
 *        with drm_connector_attach_vrr_capable_property() on connectors that
 *        could support variable refresh rates. Drivers should update the
 *        property value by calling drm_connector_set_vrr_capable_property().
 *
 *        Absence of the property should indicate absence of support.
 *
 * "VRR_ENABLED":
 *        Default &drm_crtc boolean property that notifies the driver that the
 *        content on the CRTC is suitable for variable refresh rate presentation.
 *        The driver will take this property as a hint to enable variable
 *        refresh rate support if the receiver supports it, ie. if the
 *        "vrr_capable" property is true on the &drm_connector object. The
 *        vertical front porch duration will be extended until page-flip or
 *        timeout when enabled.
 *
 *        The minimum vertical front porch duration is defined as the vertical
 *        front porch duration for the current mode.
 *
 *        The maximum vertical front porch duration is greater than or equal to
 *        the minimum vertical front porch duration. The duration is derived
 *        from the minimum supported variable refresh rate for the connector.
 *
 *        The driver may place further restrictions within these minimum
 *        and maximum bounds.
 */

/**
 * drm_connector_attach_vrr_capable_property - creates the
 * vrr_capable property
 * @connector: connector to create the vrr_capable property on.
 *
 * This is used by atomic drivers to add support for querying
 * variable refresh rate capability for a connector.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_attach_vrr_capable_property(
        struct drm_connector *connector)
{
        struct drm_device *dev = connector->dev;
        struct drm_property *prop;

        if (!connector->vrr_capable_property) {
                prop = drm_property_create_bool(dev, DRM_MODE_PROP_IMMUTABLE,
                        "vrr_capable");
                if (!prop)
                        return -ENOMEM;

                connector->vrr_capable_property = prop;
                drm_object_attach_property(&connector->base, prop, 0);
        }

        return 0;
}
EXPORT_SYMBOL(drm_connector_attach_vrr_capable_property);

/**
 * drm_connector_attach_scaling_mode_property - attach atomic scaling mode property
 * @connector: connector to attach scaling mode property on.
 * @scaling_mode_mask: or'ed mask of BIT(%DRM_MODE_SCALE_\*).
 *
 * This is used to add support for scaling mode to atomic drivers.
 * The scaling mode will be set to &drm_connector_state.scaling_mode
 * and can be used from &drm_connector_helper_funcs->atomic_check for validation.
 *
 * This is the atomic version of drm_mode_create_scaling_mode_property().
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_attach_scaling_mode_property(struct drm_connector *connector,
                                               u32 scaling_mode_mask)
{
        struct drm_device *dev = connector->dev;
        struct drm_property *scaling_mode_property;
        int i;
        const unsigned valid_scaling_mode_mask =
                (1U << ARRAY_SIZE(drm_scaling_mode_enum_list)) - 1;

        if (WARN_ON(hweight32(scaling_mode_mask) < 2 ||
                    scaling_mode_mask & ~valid_scaling_mode_mask))
                return -EINVAL;

        scaling_mode_property =
                drm_property_create(dev, DRM_MODE_PROP_ENUM, "scaling mode",
                                    hweight32(scaling_mode_mask));

        if (!scaling_mode_property)
                return -ENOMEM;

        for (i = 0; i < ARRAY_SIZE(drm_scaling_mode_enum_list); i++) {
                int ret;

                if (!(BIT(i) & scaling_mode_mask))
                        continue;

                ret = drm_property_add_enum(scaling_mode_property,
                                            drm_scaling_mode_enum_list[i].type,
                                            drm_scaling_mode_enum_list[i].name);

                if (ret) {
                        drm_property_destroy(dev, scaling_mode_property);

                        return ret;
                }
        }

        drm_object_attach_property(&connector->base,
                                   scaling_mode_property, 0);

        connector->scaling_mode_property = scaling_mode_property;

        return 0;
}
EXPORT_SYMBOL(drm_connector_attach_scaling_mode_property);

/**
 * drm_mode_create_aspect_ratio_property - create aspect ratio property
 * @dev: DRM device
 *
 * Called by a driver the first time it's needed, must be attached to desired
 * connectors.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_mode_create_aspect_ratio_property(struct drm_device *dev)
{
        if (dev->mode_config.aspect_ratio_property)
                return 0;

        dev->mode_config.aspect_ratio_property =
                drm_property_create_enum(dev, 0, "aspect ratio",
                                drm_aspect_ratio_enum_list,
                                ARRAY_SIZE(drm_aspect_ratio_enum_list));

        if (dev->mode_config.aspect_ratio_property == NULL)
                return -ENOMEM;

        return 0;
}
EXPORT_SYMBOL(drm_mode_create_aspect_ratio_property);

/**
 * DOC: standard connector properties
 *
 * Colorspace:
 *        This property is used to inform the driver about the color encoding
 *        user space configured the pixel operation properties to produce.
 *        The variants set the colorimetry, transfer characteristics, and which
 *        YCbCr conversion should be used when necessary.
 *        The transfer characteristics from HDR_OUTPUT_METADATA takes precedence
 *        over this property.
 *        User space always configures the pixel operation properties to produce
 *        full quantization range data (see the Broadcast RGB property).
 *
 *        Drivers inform the sink about what colorimetry, transfer
 *        characteristics, YCbCr conversion, and quantization range to expect
 *        (this can depend on the output mode, output format and other
 *        properties). Drivers also convert the user space provided data to what
 *        the sink expects.
 *
 *        User space has to check if the sink supports all of the possible
 *        colorimetries that the driver is allowed to pick by parsing the EDID.
 *
 *        For historical reasons this property exposes a number of variants which
 *        result in undefined behavior.
 *
 *        Default:
 *                The behavior is driver-specific.
 *
 *        BT2020_RGB:
 *
 *        BT2020_YCC:
 *                User space configures the pixel operation properties to produce
 *                RGB content with Rec. ITU-R BT.2020 colorimetry, Rec.
 *                ITU-R BT.2020 (Table 4, RGB) transfer characteristics and full
 *                quantization range.
 *                User space can use the HDR_OUTPUT_METADATA property to set the
 *                transfer characteristics to PQ (Rec. ITU-R BT.2100 Table 4) or
 *                HLG (Rec. ITU-R BT.2100 Table 5) in which case, user space
 *                configures pixel operation properties to produce content with
 *                the respective transfer characteristics.
 *                User space has to make sure the sink supports Rec.
 *                ITU-R BT.2020 R'G'B' and Rec. ITU-R BT.2020 Y'C'BC'R
 *                colorimetry.
 *                Drivers can configure the sink to use an RGB format, tell the
 *                sink to expect Rec. ITU-R BT.2020 R'G'B' colorimetry and convert
 *                to the appropriate quantization range.
 *                Drivers can configure the sink to use a YCbCr format, tell the
 *                sink to expect Rec. ITU-R BT.2020 Y'C'BC'R colorimetry, convert
 *                to YCbCr using the Rec. ITU-R BT.2020 non-constant luminance
 *                conversion matrix and convert to the appropriate quantization
 *                range.
 *                The variants BT2020_RGB and BT2020_YCC are equivalent and the
 *                driver chooses between RGB and YCbCr on its own.
 *
 *        SMPTE_170M_YCC:
 *        BT709_YCC:
 *        XVYCC_601:
 *        XVYCC_709:
 *        SYCC_601:
 *        opYCC_601:
 *        opRGB:
 *        BT2020_CYCC:
 *        DCI-P3_RGB_D65:
 *        DCI-P3_RGB_Theater:
 *        RGB_WIDE_FIXED:
 *        RGB_WIDE_FLOAT:
 *
 *        BT601_YCC:
 *                The behavior is undefined.
 *
 * Because between HDMI and DP have different colorspaces,
 * drm_mode_create_hdmi_colorspace_property() is used for HDMI connector and
 * drm_mode_create_dp_colorspace_property() is used for DP connector.
 */

static int drm_mode_create_colorspace_property(struct drm_connector *connector,
                                        u32 supported_colorspaces)
{
        struct drm_device *dev = connector->dev;
        u32 colorspaces = supported_colorspaces | BIT(DRM_MODE_COLORIMETRY_DEFAULT);
        struct drm_prop_enum_list enum_list[DRM_MODE_COLORIMETRY_COUNT];
        int i, len;

        if (connector->colorspace_property)
                return 0;

        if (!supported_colorspaces) {
                drm_err(dev, "No supported colorspaces provded on [CONNECTOR:%d:%s]\n",
                            connector->base.id, connector->name);
                return -EINVAL;
        }

        if ((supported_colorspaces & -BIT(DRM_MODE_COLORIMETRY_COUNT)) != 0) {
                drm_err(dev, "Unknown colorspace provded on [CONNECTOR:%d:%s]\n",
                            connector->base.id, connector->name);
                return -EINVAL;
        }

        len = 0;
        for (i = 0; i < DRM_MODE_COLORIMETRY_COUNT; i++) {
                if ((colorspaces & BIT(i)) == 0)
                        continue;

                enum_list[len].type = i;
                enum_list[len].name = colorspace_names[i];
                len++;
        }

        connector->colorspace_property =
                drm_property_create_enum(dev, DRM_MODE_PROP_ENUM, "Colorspace",
                                        enum_list,
                                        len);

        if (!connector->colorspace_property)
                return -ENOMEM;

        return 0;
}

/**
 * drm_mode_create_hdmi_colorspace_property - create hdmi colorspace property
 * @connector: connector to create the Colorspace property on.
 * @supported_colorspaces: bitmap of supported color spaces
 *
 * Called by a driver the first time it's needed, must be attached to desired
 * HDMI connectors.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_mode_create_hdmi_colorspace_property(struct drm_connector *connector,
                                             u32 supported_colorspaces)
{
        u32 colorspaces;

        if (supported_colorspaces)
                colorspaces = supported_colorspaces & hdmi_colorspaces;
        else
                colorspaces = hdmi_colorspaces;

        return drm_mode_create_colorspace_property(connector, colorspaces);
}
EXPORT_SYMBOL(drm_mode_create_hdmi_colorspace_property);

/**
 * drm_mode_create_dp_colorspace_property - create dp colorspace property
 * @connector: connector to create the Colorspace property on.
 * @supported_colorspaces: bitmap of supported color spaces
 *
 * Called by a driver the first time it's needed, must be attached to desired
 * DP connectors.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_mode_create_dp_colorspace_property(struct drm_connector *connector,
                                           u32 supported_colorspaces)
{
        u32 colorspaces;

        if (supported_colorspaces)
                colorspaces = supported_colorspaces & dp_colorspaces;
        else
                colorspaces = dp_colorspaces;

        return drm_mode_create_colorspace_property(connector, colorspaces);
}
EXPORT_SYMBOL(drm_mode_create_dp_colorspace_property);

/**
 * drm_mode_create_content_type_property - create content type property
 * @dev: DRM device
 *
 * Called by a driver the first time it's needed, must be attached to desired
 * connectors.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_mode_create_content_type_property(struct drm_device *dev)
{
        if (dev->mode_config.content_type_property)
                return 0;

        dev->mode_config.content_type_property =
                drm_property_create_enum(dev, 0, "content type",
                                         drm_content_type_enum_list,
                                         ARRAY_SIZE(drm_content_type_enum_list));

        if (dev->mode_config.content_type_property == NULL)
                return -ENOMEM;

        return 0;
}
EXPORT_SYMBOL(drm_mode_create_content_type_property);

/**
 * drm_mode_create_suggested_offset_properties - create suggests offset properties
 * @dev: DRM device
 *
 * Create the suggested x/y offset property for connectors.
 *
 * Returns:
 * 0 on success or a negative error code on failure.
 */
int drm_mode_create_suggested_offset_properties(struct drm_device *dev)
{
        if (dev->mode_config.suggested_x_property && dev->mode_config.suggested_y_property)
                return 0;

        dev->mode_config.suggested_x_property =
                drm_property_create_range(dev, DRM_MODE_PROP_IMMUTABLE, "suggested X", 0, 0xffffffff);

        dev->mode_config.suggested_y_property =
                drm_property_create_range(dev, DRM_MODE_PROP_IMMUTABLE, "suggested Y", 0, 0xffffffff);

        if (dev->mode_config.suggested_x_property == NULL ||
            dev->mode_config.suggested_y_property == NULL)
                return -ENOMEM;
        return 0;
}
EXPORT_SYMBOL(drm_mode_create_suggested_offset_properties);

/**
 * drm_connector_set_path_property - set tile property on connector
 * @connector: connector to set property on.
 * @path: path to use for property; must not be NULL.
 *
 * This creates a property to expose to userspace to specify a
 * connector path. This is mainly used for DisplayPort MST where
 * connectors have a topology and we want to allow userspace to give
 * them more meaningful names.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_set_path_property(struct drm_connector *connector,
                                    const char *path)
{
        struct drm_device *dev = connector->dev;
        int ret;

        ret = drm_property_replace_global_blob(dev,
                                               &connector->path_blob_ptr,
                                               strlen(path) + 1,
                                               path,
                                               &connector->base,
                                               dev->mode_config.path_property);
        return ret;
}
EXPORT_SYMBOL(drm_connector_set_path_property);

/**
 * drm_connector_set_tile_property - set tile property on connector
 * @connector: connector to set property on.
 *
 * This looks up the tile information for a connector, and creates a
 * property for userspace to parse if it exists. The property is of
 * the form of 8 integers using ':' as a separator.
 * This is used for dual port tiled displays with DisplayPort SST
 * or DisplayPort MST connectors.
 *
 * Returns:
 * Zero on success, errno on failure.
 */
int drm_connector_set_tile_property(struct drm_connector *connector)
{
        struct drm_device *dev = connector->dev;
        char tile[256];
        int ret;

        if (!connector->has_tile) {
                ret  = drm_property_replace_global_blob(dev,
                                                        &connector->tile_blob_ptr,
                                                        0,
                                                        NULL,
                                                        &connector->base,
                                                        dev->mode_config.tile_property);
                return ret;
        }

        snprintf(tile, 256, "%d:%d:%d:%d:%d:%d:%d:%d",
                 connector->tile_group->id, connector->tile_is_single_monitor,
                 connector->num_h_tile, connector->num_v_tile,
                 connector->tile_h_loc, connector->tile_v_loc,
                 connector->tile_h_size, connector->tile_v_size);

        ret = drm_property_replace_global_blob(dev,
                                               &connector->tile_blob_ptr,
                                               strlen(tile) + 1,
                                               tile,
                                               &connector->base,
                                               dev->mode_config.tile_property);
        return ret;
}
EXPORT_SYMBOL(drm_connector_set_tile_property);

/**
 * drm_connector_set_link_status_property - Set link status property of a connector
 * @connector: drm connector
 * @link_status: new value of link status property (0: Good, 1: Bad)
 *
 * In usual working scenario, this link status property will always be set to
 * "GOOD". If something fails during or after a mode set, the kernel driver
 * may set this link status property to "BAD". The caller then needs to send a
 * hotplug uevent for userspace to re-check the valid modes through
 * GET_CONNECTOR_IOCTL and retry modeset.
 *
 * Note: Drivers cannot rely on userspace to support this property and
 * issue a modeset. As such, they may choose to handle issues (like
 * re-training a link) without userspace's intervention.
 *
 * The reason for adding this property is to handle link training failures, but
 * it is not limited to DP or link training. For example, if we implement
 * asynchronous setcrtc, this property can be used to report any failures in that.
 */
void drm_connector_set_link_status_property(struct drm_connector *connector,
                                            uint64_t link_status)
{
        struct drm_device *dev = connector->dev;

        drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
        connector->state->link_status = link_status;
        drm_modeset_unlock(&dev->mode_config.connection_mutex);
}
EXPORT_SYMBOL(drm_connector_set_link_status_property);

/**
 * drm_connector_attach_max_bpc_property - attach "max bpc" property
 * @connector: connector to attach max bpc property on.
 * @min: The minimum bit depth supported by the connector.
 * @max: The maximum bit depth supported by the connector.
 *
 * This is used to add support for limiting the bit depth on a connector.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_attach_max_bpc_property(struct drm_connector *connector,
                                          int min, int max)
{
        struct drm_device *dev = connector->dev;
        struct drm_property *prop;

        prop = connector->max_bpc_property;
        if (!prop) {
                prop = drm_property_create_range(dev, 0, "max bpc", min, max);
                if (!prop)
                        return -ENOMEM;

                connector->max_bpc_property = prop;
        }

        drm_object_attach_property(&connector->base, prop, max);
        connector->state->max_requested_bpc = max;
        connector->state->max_bpc = max;

        return 0;
}
EXPORT_SYMBOL(drm_connector_attach_max_bpc_property);

/**
 * drm_connector_attach_hdr_output_metadata_property - attach "HDR_OUTPUT_METADA" property
 * @connector: connector to attach the property on.
 *
 * This is used to allow the userspace to send HDR Metadata to the
 * driver.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_attach_hdr_output_metadata_property(struct drm_connector *connector)
{
        struct drm_device *dev = connector->dev;
        struct drm_property *prop = dev->mode_config.hdr_output_metadata_property;

        drm_object_attach_property(&connector->base, prop, 0);

        return 0;
}
EXPORT_SYMBOL(drm_connector_attach_hdr_output_metadata_property);

/**
 * drm_connector_attach_broadcast_rgb_property - attach "Broadcast RGB" property
 * @connector: connector to attach the property on.
 *
 * This is used to add support for forcing the RGB range on a connector
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_attach_broadcast_rgb_property(struct drm_connector *connector)
{
        struct drm_device *dev = connector->dev;
        struct drm_property *prop;

        prop = connector->broadcast_rgb_property;
        if (!prop) {
                prop = drm_property_create_enum(dev, DRM_MODE_PROP_ENUM,
                                                "Broadcast RGB",
                                                broadcast_rgb_names,
                                                ARRAY_SIZE(broadcast_rgb_names));
                if (!prop)
                        return -EINVAL;

                connector->broadcast_rgb_property = prop;
        }

        drm_object_attach_property(&connector->base, prop,
                                   DRM_HDMI_BROADCAST_RGB_AUTO);

        return 0;
}
EXPORT_SYMBOL(drm_connector_attach_broadcast_rgb_property);

/**
 * drm_connector_attach_colorspace_property - attach "Colorspace" property
 * @connector: connector to attach the property on.
 *
 * This is used to allow the userspace to signal the output colorspace
 * to the driver.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_attach_colorspace_property(struct drm_connector *connector)
{
        struct drm_property *prop = connector->colorspace_property;

        drm_object_attach_property(&connector->base, prop, DRM_MODE_COLORIMETRY_DEFAULT);

        return 0;
}
EXPORT_SYMBOL(drm_connector_attach_colorspace_property);

/**
 * drm_connector_atomic_hdr_metadata_equal - checks if the hdr metadata changed
 * @old_state: old connector state to compare
 * @new_state: new connector state to compare
 *
 * This is used by HDR-enabled drivers to test whether the HDR metadata
 * have changed between two different connector state (and thus probably
 * requires a full blown mode change).
 *
 * Returns:
 * True if the metadata are equal, False otherwise
 */
bool drm_connector_atomic_hdr_metadata_equal(struct drm_connector_state *old_state,
                                             struct drm_connector_state *new_state)
{
        struct drm_property_blob *old_blob = old_state->hdr_output_metadata;
        struct drm_property_blob *new_blob = new_state->hdr_output_metadata;

        if (!old_blob || !new_blob)
                return old_blob == new_blob;

        if (old_blob->length != new_blob->length)
                return false;

        return !memcmp(old_blob->data, new_blob->data, old_blob->length);
}
EXPORT_SYMBOL(drm_connector_atomic_hdr_metadata_equal);

/**
 * drm_connector_set_vrr_capable_property - sets the variable refresh rate
 * capable property for a connector
 * @connector: drm connector
 * @capable: True if the connector is variable refresh rate capable
 *
 * Should be used by atomic drivers to update the indicated support for
 * variable refresh rate over a connector.
 */
void drm_connector_set_vrr_capable_property(
                struct drm_connector *connector, bool capable)
{
        if (!connector->vrr_capable_property)
                return;

        drm_object_property_set_value(&connector->base,
                                      connector->vrr_capable_property,
                                      capable);
}
EXPORT_SYMBOL(drm_connector_set_vrr_capable_property);

/**
 * drm_connector_set_panel_orientation - sets the connector's panel_orientation
 * @connector: connector for which to set the panel-orientation property.
 * @panel_orientation: drm_panel_orientation value to set
 *
 * This function sets the connector's panel_orientation and attaches
 * a "panel orientation" property to the connector.
 *
 * Calling this function on a connector where the panel_orientation has
 * already been set is a no-op (e.g. the orientation has been overridden with
 * a kernel commandline option).
 *
 * It is allowed to call this function with a panel_orientation of
 * DRM_MODE_PANEL_ORIENTATION_UNKNOWN, in which case it is a no-op.
 *
 * The function shouldn't be called in panel after drm is registered (i.e.
 * drm_dev_register() is called in drm).
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_set_panel_orientation(
        struct drm_connector *connector,
        enum drm_panel_orientation panel_orientation)
{
        struct drm_device *dev = connector->dev;
        struct drm_display_info *info = &connector->display_info;
        struct drm_property *prop;

        /* Already set? */
        if (info->panel_orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
                return 0;

        /* Don't attach the property if the orientation is unknown */
        if (panel_orientation == DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
                return 0;

        info->panel_orientation = panel_orientation;

        prop = dev->mode_config.panel_orientation_property;
        if (!prop) {
                prop = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
                                "panel orientation",
                                drm_panel_orientation_enum_list,
                                ARRAY_SIZE(drm_panel_orientation_enum_list));
                if (!prop)
                        return -ENOMEM;

                dev->mode_config.panel_orientation_property = prop;
        }

        drm_object_attach_property(&connector->base, prop,
                                   info->panel_orientation);
        return 0;
}
EXPORT_SYMBOL(drm_connector_set_panel_orientation);

/**
 * drm_connector_set_panel_orientation_with_quirk - set the
 *        connector's panel_orientation after checking for quirks
 * @connector: connector for which to init the panel-orientation property.
 * @panel_orientation: drm_panel_orientation value to set
 * @width: width in pixels of the panel, used for panel quirk detection
 * @height: height in pixels of the panel, used for panel quirk detection
 *
 * Like drm_connector_set_panel_orientation(), but with a check for platform
 * specific (e.g. DMI based) quirks overriding the passed in panel_orientation.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_set_panel_orientation_with_quirk(
        struct drm_connector *connector,
        enum drm_panel_orientation panel_orientation,
        int width, int height)
{
        int orientation_quirk;

        orientation_quirk = drm_get_panel_orientation_quirk(width, height);
        if (orientation_quirk != DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
                panel_orientation = orientation_quirk;

        return drm_connector_set_panel_orientation(connector,
                                                   panel_orientation);
}
EXPORT_SYMBOL(drm_connector_set_panel_orientation_with_quirk);

/**
 * drm_connector_set_orientation_from_panel -
 *        set the connector's panel_orientation from panel's callback.
 * @connector: connector for which to init the panel-orientation property.
 * @panel: panel that can provide orientation information.
 *
 * Drm drivers should call this function before drm_dev_register().
 * Orientation is obtained from panel's .get_orientation() callback.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_connector_set_orientation_from_panel(
        struct drm_connector *connector,
        struct drm_panel *panel)
{
        enum drm_panel_orientation orientation;

        if (panel && panel->funcs && panel->funcs->get_orientation)
                orientation = panel->funcs->get_orientation(panel);
        else
                orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN;

        return drm_connector_set_panel_orientation(connector, orientation);
}
EXPORT_SYMBOL(drm_connector_set_orientation_from_panel);

static const struct drm_prop_enum_list privacy_screen_enum[] = {
        { PRIVACY_SCREEN_DISABLED,                "Disabled" },
        { PRIVACY_SCREEN_ENABLED,                "Enabled" },
        { PRIVACY_SCREEN_DISABLED_LOCKED,        "Disabled-locked" },
        { PRIVACY_SCREEN_ENABLED_LOCKED,        "Enabled-locked" },
};

/**
 * drm_connector_create_privacy_screen_properties - create the drm connecter's
 *    privacy-screen properties.
 * @connector: connector for which to create the privacy-screen properties
 *
 * This function creates the "privacy-screen sw-state" and "privacy-screen
 * hw-state" properties for the connector. They are not attached.
 */
void
drm_connector_create_privacy_screen_properties(struct drm_connector *connector)
{
        if (connector->privacy_screen_sw_state_property)
                return;

        /* Note sw-state only supports the first 2 values of the enum */
        connector->privacy_screen_sw_state_property =
                drm_property_create_enum(connector->dev, DRM_MODE_PROP_ENUM,
                                "privacy-screen sw-state",
                                privacy_screen_enum, 2);

        connector->privacy_screen_hw_state_property =
                drm_property_create_enum(connector->dev,
                                DRM_MODE_PROP_IMMUTABLE | DRM_MODE_PROP_ENUM,
                                "privacy-screen hw-state",
                                privacy_screen_enum,
                                ARRAY_SIZE(privacy_screen_enum));
}
EXPORT_SYMBOL(drm_connector_create_privacy_screen_properties);

/**
 * drm_connector_attach_privacy_screen_properties - attach the drm connecter's
 *    privacy-screen properties.
 * @connector: connector on which to attach the privacy-screen properties
 *
 * This function attaches the "privacy-screen sw-state" and "privacy-screen
 * hw-state" properties to the connector. The initial state of both is set
 * to "Disabled".
 */
void
drm_connector_attach_privacy_screen_properties(struct drm_connector *connector)
{
        if (!connector->privacy_screen_sw_state_property)
                return;

        drm_object_attach_property(&connector->base,
                                   connector->privacy_screen_sw_state_property,
                                   PRIVACY_SCREEN_DISABLED);

        drm_object_attach_property(&connector->base,
                                   connector->privacy_screen_hw_state_property,
                                   PRIVACY_SCREEN_DISABLED);
}
EXPORT_SYMBOL(drm_connector_attach_privacy_screen_properties);

static void drm_connector_update_privacy_screen_properties(
        struct drm_connector *connector, bool set_sw_state)
{
        enum drm_privacy_screen_status sw_state, hw_state;

        drm_privacy_screen_get_state(connector->privacy_screen,
                                     &sw_state, &hw_state);

        if (set_sw_state)
                connector->state->privacy_screen_sw_state = sw_state;
        drm_object_property_set_value(&connector->base,
                        connector->privacy_screen_hw_state_property, hw_state);
}

static int drm_connector_privacy_screen_notifier(
        struct notifier_block *nb, unsigned long action, void *data)
{
        struct drm_connector *connector =
                container_of(nb, struct drm_connector, privacy_screen_notifier);
        struct drm_device *dev = connector->dev;

        drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
        drm_connector_update_privacy_screen_properties(connector, true);
        drm_modeset_unlock(&dev->mode_config.connection_mutex);

        drm_sysfs_connector_property_event(connector,
                                           connector->privacy_screen_sw_state_property);
        drm_sysfs_connector_property_event(connector,
                                           connector->privacy_screen_hw_state_property);

        return NOTIFY_DONE;
}

/**
 * drm_connector_attach_privacy_screen_provider - attach a privacy-screen to
 *    the connector
 * @connector: connector to attach the privacy-screen to
 * @priv: drm_privacy_screen to attach
 *
 * Create and attach the standard privacy-screen properties and register
 * a generic notifier for generating sysfs-connector-status-events
 * on external changes to the privacy-screen status.
 * This function takes ownership of the passed in drm_privacy_screen and will
 * call drm_privacy_screen_put() on it when the connector is destroyed.
 */
void drm_connector_attach_privacy_screen_provider(
        struct drm_connector *connector, struct drm_privacy_screen *priv)
{
        connector->privacy_screen = priv;
        connector->privacy_screen_notifier.notifier_call =
                drm_connector_privacy_screen_notifier;

        drm_connector_create_privacy_screen_properties(connector);
        drm_connector_update_privacy_screen_properties(connector, true);
        drm_connector_attach_privacy_screen_properties(connector);
}
EXPORT_SYMBOL(drm_connector_attach_privacy_screen_provider);

/**
 * drm_connector_update_privacy_screen - update connector's privacy-screen sw-state
 * @connector_state: connector-state to update the privacy-screen for
 *
 * This function calls drm_privacy_screen_set_sw_state() on the connector's
 * privacy-screen.
 *
 * If the connector has no privacy-screen, then this is a no-op.
 */
void drm_connector_update_privacy_screen(const struct drm_connector_state *connector_state)
{
        struct drm_connector *connector = connector_state->connector;
        int ret;

        if (!connector->privacy_screen)
                return;

        ret = drm_privacy_screen_set_sw_state(connector->privacy_screen,
                                              connector_state->privacy_screen_sw_state);
        if (ret) {
                drm_err(connector->dev, "Error updating privacy-screen sw_state\n");
                return;
        }

        /* The hw_state property value may have changed, update it. */
        drm_connector_update_privacy_screen_properties(connector, false);
}
EXPORT_SYMBOL(drm_connector_update_privacy_screen);

int drm_connector_set_obj_prop(struct drm_mode_object *obj,
                                    struct drm_property *property,
                                    uint64_t value)
{
        int ret = -EINVAL;
        struct drm_connector *connector = obj_to_connector(obj);

        /* Do DPMS ourselves */
        if (property == connector->dev->mode_config.dpms_property) {
                ret = (*connector->funcs->dpms)(connector, (int)value);
        } else if (connector->funcs->set_property)
                ret = connector->funcs->set_property(connector, property, value);

        if (!ret)
                drm_object_property_set_value(&connector->base, property, value);
        return ret;
}

int drm_connector_property_set_ioctl(struct drm_device *dev,
                                     void *data, struct drm_file *file_priv)
{
        struct drm_mode_connector_set_property *conn_set_prop = data;
        struct drm_mode_obj_set_property obj_set_prop = {
                .value = conn_set_prop->value,
                .prop_id = conn_set_prop->prop_id,
                .obj_id = conn_set_prop->connector_id,
                .obj_type = DRM_MODE_OBJECT_CONNECTOR
        };

        /* It does all the locking and checking we need */
        return drm_mode_obj_set_property_ioctl(dev, &obj_set_prop, file_priv);
}

static struct drm_encoder *drm_connector_get_encoder(struct drm_connector *connector)
{
        /* For atomic drivers only state objects are synchronously updated and
         * protected by modeset locks, so check those first.
         */
        if (connector->state)
                return connector->state->best_encoder;
        return connector->encoder;
}

static bool
drm_mode_expose_to_userspace(const struct drm_display_mode *mode,
                             const struct list_head *modes,
                             const struct drm_file *file_priv)
{
        /*
         * If user-space hasn't configured the driver to expose the stereo 3D
         * modes, don't expose them.
         */
        if (!file_priv->stereo_allowed && drm_mode_is_stereo(mode))
                return false;
        /*
         * If user-space hasn't configured the driver to expose the modes
         * with aspect-ratio, don't expose them. However if such a mode
         * is unique, let it be exposed, but reset the aspect-ratio flags
         * while preparing the list of user-modes.
         */
        if (!file_priv->aspect_ratio_allowed) {
                const struct drm_display_mode *mode_itr;

                list_for_each_entry(mode_itr, modes, head) {
                        if (mode_itr->expose_to_userspace &&
                            drm_mode_match(mode_itr, mode,
                                           DRM_MODE_MATCH_TIMINGS |
                                           DRM_MODE_MATCH_CLOCK |
                                           DRM_MODE_MATCH_FLAGS |
                                           DRM_MODE_MATCH_3D_FLAGS))
                                return false;
                }
        }

        return true;
}

int drm_mode_getconnector(struct drm_device *dev, void *data,
                          struct drm_file *file_priv)
{
        struct drm_mode_get_connector *out_resp = data;
        struct drm_connector *connector;
        struct drm_encoder *encoder;
        struct drm_display_mode *mode;
        int mode_count = 0;
        int encoders_count = 0;
        int ret = 0;
        int copied = 0;
        struct drm_mode_modeinfo u_mode;
        struct drm_mode_modeinfo __user *mode_ptr;
        uint32_t __user *encoder_ptr;
        bool is_current_master;

        if (!drm_core_check_feature(dev, DRIVER_MODESET))
                return -EOPNOTSUPP;

        memset(&u_mode, 0, sizeof(struct drm_mode_modeinfo));

        connector = drm_connector_lookup(dev, file_priv, out_resp->connector_id);
        if (!connector)
                return -ENOENT;

        encoders_count = hweight32(connector->possible_encoders);

        if ((out_resp->count_encoders >= encoders_count) && encoders_count) {
                copied = 0;
                encoder_ptr = (uint32_t __user *)(unsigned long)(out_resp->encoders_ptr);

                drm_connector_for_each_possible_encoder(connector, encoder) {
                        if (put_user(encoder->base.id, encoder_ptr + copied)) {
                                ret = -EFAULT;
                                goto out;
                        }
                        copied++;
                }
        }
        out_resp->count_encoders = encoders_count;

        out_resp->connector_id = connector->base.id;
        out_resp->connector_type = connector->connector_type;
        out_resp->connector_type_id = connector->connector_type_id;

        is_current_master = drm_is_current_master(file_priv);

        mutex_lock(&dev->mode_config.mutex);
        if (out_resp->count_modes == 0) {
                if (is_current_master)
                        connector->funcs->fill_modes(connector,
                                                     dev->mode_config.max_width,
                                                     dev->mode_config.max_height);
                else
                        drm_dbg_kms(dev, "User-space requested a forced probe on [CONNECTOR:%d:%s] but is not the DRM master, demoting to read-only probe\n",
                                    connector->base.id, connector->name);
        }

        out_resp->mm_width = connector->display_info.width_mm;
        out_resp->mm_height = connector->display_info.height_mm;
        out_resp->subpixel = connector->display_info.subpixel_order;
        out_resp->connection = connector->status;

        /* delayed so we get modes regardless of pre-fill_modes state */
        list_for_each_entry(mode, &connector->modes, head) {
                WARN_ON(mode->expose_to_userspace);

                if (drm_mode_expose_to_userspace(mode, &connector->modes,
                                                 file_priv)) {
                        mode->expose_to_userspace = true;
                        mode_count++;
                }
        }

        /*
         * This ioctl is called twice, once to determine how much space is
         * needed, and the 2nd time to fill it.
         */
        if ((out_resp->count_modes >= mode_count) && mode_count) {
                copied = 0;
                mode_ptr = (struct drm_mode_modeinfo __user *)(unsigned long)out_resp->modes_ptr;
                list_for_each_entry(mode, &connector->modes, head) {
                        if (!mode->expose_to_userspace)
                                continue;

                        /* Clear the tag for the next time around */
                        mode->expose_to_userspace = false;

                        drm_mode_convert_to_umode(&u_mode, mode);
                        /*
                         * Reset aspect ratio flags of user-mode, if modes with
                         * aspect-ratio are not supported.
                         */
                        if (!file_priv->aspect_ratio_allowed)
                                u_mode.flags &= ~DRM_MODE_FLAG_PIC_AR_MASK;
                        if (copy_to_user(mode_ptr + copied,
                                         &u_mode, sizeof(u_mode))) {
                                ret = -EFAULT;

                                /*
                                 * Clear the tag for the rest of
                                 * the modes for the next time around.
                                 */
                                list_for_each_entry_continue(mode, &connector->modes, head)
                                        mode->expose_to_userspace = false;

                                mutex_unlock(&dev->mode_config.mutex);

                                goto out;
                        }
                        copied++;
                }
        } else {
                /* Clear the tag for the next time around */
                list_for_each_entry(mode, &connector->modes, head)
                        mode->expose_to_userspace = false;
        }

        out_resp->count_modes = mode_count;
        mutex_unlock(&dev->mode_config.mutex);

        drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
        encoder = drm_connector_get_encoder(connector);
        if (encoder)
                out_resp->encoder_id = encoder->base.id;
        else
                out_resp->encoder_id = 0;

        /* Only grab properties after probing, to make sure EDID and other
         * properties reflect the latest status.
         */
        ret = drm_mode_object_get_properties(&connector->base, file_priv->atomic,
                        (uint32_t __user *)(unsigned long)(out_resp->props_ptr),
                        (uint64_t __user *)(unsigned long)(out_resp->prop_values_ptr),
                        &out_resp->count_props);
        drm_modeset_unlock(&dev->mode_config.connection_mutex);

out:
        drm_connector_put(connector);

        return ret;
}

/**
 * drm_connector_find_by_fwnode - Find a connector based on the associated fwnode
 * @fwnode: fwnode for which to find the matching drm_connector
 *
 * This functions looks up a drm_connector based on its associated fwnode. When
 * a connector is found a reference to the connector is returned. The caller must
 * call drm_connector_put() to release this reference when it is done with the
 * connector.
 *
 * Returns: A reference to the found connector or an ERR_PTR().
 */
struct drm_connector *drm_connector_find_by_fwnode(struct fwnode_handle *fwnode)
{
        struct drm_connector *connector, *found = ERR_PTR(-ENODEV);

        if (!fwnode)
                return ERR_PTR(-ENODEV);

        mutex_lock(&connector_list_lock);

        list_for_each_entry(connector, &connector_list, global_connector_list_entry) {
                if (connector->fwnode == fwnode ||
                    (connector->fwnode && connector->fwnode->secondary == fwnode)) {
                        drm_connector_get(connector);
                        found = connector;
                        break;
                }
        }

        mutex_unlock(&connector_list_lock);

        return found;
}

/**
 * drm_connector_oob_hotplug_event - Report out-of-band hotplug event to connector
 * @connector_fwnode: fwnode_handle to report the event on
 * @status: hot plug detect logical state
 *
 * On some hardware a hotplug event notification may come from outside the display
 * driver / device. An example of this is some USB Type-C setups where the hardware
 * muxes the DisplayPort data and aux-lines but does not pass the altmode HPD
 * status bit to the GPU's DP HPD pin.
 *
 * This function can be used to report these out-of-band events after obtaining
 * a drm_connector reference through calling drm_connector_find_by_fwnode().
 */
void drm_connector_oob_hotplug_event(struct fwnode_handle *connector_fwnode,
                                     enum drm_connector_status status)
{
        struct drm_connector *connector;

        connector = drm_connector_find_by_fwnode(connector_fwnode);
        if (IS_ERR(connector))
                return;

        if (connector->funcs->oob_hotplug_event)
                connector->funcs->oob_hotplug_event(connector, status);

        drm_connector_put(connector);
}
EXPORT_SYMBOL(drm_connector_oob_hotplug_event);


/**
 * DOC: Tile group
 *
 * Tile groups are used to represent tiled monitors with a unique integer
 * identifier. Tiled monitors using DisplayID v1.3 have a unique 8-byte handle,
 * we store this in a tile group, so we have a common identifier for all tiles
 * in a monitor group. The property is called "TILE". Drivers can manage tile
 * groups using drm_mode_create_tile_group(), drm_mode_put_tile_group() and
 * drm_mode_get_tile_group(). But this is only needed for internal panels where
 * the tile group information is exposed through a non-standard way.
 */

static void drm_tile_group_free(struct kref *kref)
{
        struct drm_tile_group *tg = container_of(kref, struct drm_tile_group, refcount);
        struct drm_device *dev = tg->dev;

        mutex_lock(&dev->mode_config.idr_mutex);
        idr_remove(&dev->mode_config.tile_idr, tg->id);
        mutex_unlock(&dev->mode_config.idr_mutex);
        kfree(tg);
}

/**
 * drm_mode_put_tile_group - drop a reference to a tile group.
 * @dev: DRM device
 * @tg: tile group to drop reference to.
 *
 * drop reference to tile group and free if 0.
 */
void drm_mode_put_tile_group(struct drm_device *dev,
                             struct drm_tile_group *tg)
{
        kref_put(&tg->refcount, drm_tile_group_free);
}
EXPORT_SYMBOL(drm_mode_put_tile_group);

/**
 * drm_mode_get_tile_group - get a reference to an existing tile group
 * @dev: DRM device
 * @topology: 8-bytes unique per monitor.
 *
 * Use the unique bytes to get a reference to an existing tile group.
 *
 * RETURNS:
 * tile group or NULL if not found.
 */
struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev,
                                               const char topology[8])
{
        struct drm_tile_group *tg;
        int id;

        mutex_lock(&dev->mode_config.idr_mutex);
        idr_for_each_entry(&dev->mode_config.tile_idr, tg, id) {
                if (!memcmp(tg->group_data, topology, 8)) {
                        if (!kref_get_unless_zero(&tg->refcount))
                                tg = NULL;
                        mutex_unlock(&dev->mode_config.idr_mutex);
                        return tg;
                }
        }
        mutex_unlock(&dev->mode_config.idr_mutex);
        return NULL;
}
EXPORT_SYMBOL(drm_mode_get_tile_group);

/**
 * drm_mode_create_tile_group - create a tile group from a displayid description
 * @dev: DRM device
 * @topology: 8-bytes unique per monitor.
 *
 * Create a tile group for the unique monitor, and get a unique
 * identifier for the tile group.
 *
 * RETURNS:
 * new tile group or NULL.
 */
struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
                                                  const char topology[8])
{
        struct drm_tile_group *tg;
        int ret;

        tg = kzalloc(sizeof(*tg), GFP_KERNEL);
        if (!tg)
                return NULL;

        kref_init(&tg->refcount);
        memcpy(tg->group_data, topology, 8);
        tg->dev = dev;

        mutex_lock(&dev->mode_config.idr_mutex);
        ret = idr_alloc(&dev->mode_config.tile_idr, tg, 1, 0, GFP_KERNEL);
        if (ret >= 0) {
                tg->id = ret;
        } else {
                kfree(tg);
                tg = NULL;
        }

        mutex_unlock(&dev->mode_config.idr_mutex);
        return tg;
}
EXPORT_SYMBOL(drm_mode_create_tile_group);



























































    1 































    1 

























































    1 




























    1 





























    1 






























    1 
































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
// SPDX-License-Identifier: GPL-2.0-only

#include <linux/phy.h>
#include <linux/ethtool_netlink.h>

#include "netlink.h"
#include "common.h"

struct plca_req_info {
        struct ethnl_req_info                base;
};

struct plca_reply_data {
        struct ethnl_reply_data                base;
        struct phy_plca_cfg                plca_cfg;
        struct phy_plca_status                plca_st;
};

// Helpers ------------------------------------------------------------------ //

#define PLCA_REPDATA(__reply_base) \
        container_of(__reply_base, struct plca_reply_data, base)

// PLCA get configuration message ------------------------------------------- //

const struct nla_policy ethnl_plca_get_cfg_policy[] = {
        [ETHTOOL_A_PLCA_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy_phy),
};

static void plca_update_sint(int *dst, struct nlattr **tb, u32 attrid,
                             bool *mod)
{
        const struct nlattr *attr = tb[attrid];

        if (!attr ||
            WARN_ON_ONCE(attrid >= ARRAY_SIZE(ethnl_plca_set_cfg_policy)))
                return;

        switch (ethnl_plca_set_cfg_policy[attrid].type) {
        case NLA_U8:
                *dst = nla_get_u8(attr);
                break;
        case NLA_U32:
                *dst = nla_get_u32(attr);
                break;
        default:
                WARN_ON_ONCE(1);
        }

        *mod = true;
}

static int plca_get_cfg_prepare_data(const struct ethnl_req_info *req_base,
                                     struct ethnl_reply_data *reply_base,
                                     const struct genl_info *info)
{
        struct plca_reply_data *data = PLCA_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        const struct ethtool_phy_ops *ops;
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        int ret;

        phydev = ethnl_req_get_phydev(req_base, tb[ETHTOOL_A_PLCA_HEADER],
                                      info->extack);
        // check that the PHY device is available and connected
        if (IS_ERR_OR_NULL(phydev)) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        // note: rtnl_lock is held already by ethnl_default_doit
        ops = ethtool_phy_ops;
        if (!ops || !ops->get_plca_cfg) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                goto out;

        memset(&data->plca_cfg, 0xff,
               sizeof_field(struct plca_reply_data, plca_cfg));

        ret = ops->get_plca_cfg(phydev, &data->plca_cfg);
        ethnl_ops_complete(dev);

out:
        return ret;
}

static int plca_get_cfg_reply_size(const struct ethnl_req_info *req_base,
                                   const struct ethnl_reply_data *reply_base)
{
        return nla_total_size(sizeof(u16)) +        /* _VERSION */
               nla_total_size(sizeof(u8)) +        /* _ENABLED */
               nla_total_size(sizeof(u32)) +        /* _NODE_CNT */
               nla_total_size(sizeof(u32)) +        /* _NODE_ID */
               nla_total_size(sizeof(u32)) +        /* _TO_TIMER */
               nla_total_size(sizeof(u32)) +        /* _BURST_COUNT */
               nla_total_size(sizeof(u32));        /* _BURST_TIMER */
}

static int plca_get_cfg_fill_reply(struct sk_buff *skb,
                                   const struct ethnl_req_info *req_base,
                                   const struct ethnl_reply_data *reply_base)
{
        const struct plca_reply_data *data = PLCA_REPDATA(reply_base);
        const struct phy_plca_cfg *plca = &data->plca_cfg;

        if ((plca->version >= 0 &&
             nla_put_u16(skb, ETHTOOL_A_PLCA_VERSION, plca->version)) ||
            (plca->enabled >= 0 &&
             nla_put_u8(skb, ETHTOOL_A_PLCA_ENABLED, !!plca->enabled)) ||
            (plca->node_id >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_NODE_ID, plca->node_id)) ||
            (plca->node_cnt >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_NODE_CNT, plca->node_cnt)) ||
            (plca->to_tmr >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_TO_TMR, plca->to_tmr)) ||
            (plca->burst_cnt >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_BURST_CNT, plca->burst_cnt)) ||
            (plca->burst_tmr >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_BURST_TMR, plca->burst_tmr)))
                return -EMSGSIZE;

        return 0;
};

// PLCA set configuration message ------------------------------------------- //

const struct nla_policy ethnl_plca_set_cfg_policy[] = {
        [ETHTOOL_A_PLCA_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy_phy),
        [ETHTOOL_A_PLCA_ENABLED]        = NLA_POLICY_MAX(NLA_U8, 1),
        [ETHTOOL_A_PLCA_NODE_ID]        = NLA_POLICY_MAX(NLA_U32, 255),
        [ETHTOOL_A_PLCA_NODE_CNT]        = NLA_POLICY_RANGE(NLA_U32, 1, 255),
        [ETHTOOL_A_PLCA_TO_TMR]                = NLA_POLICY_MAX(NLA_U32, 255),
        [ETHTOOL_A_PLCA_BURST_CNT]        = NLA_POLICY_MAX(NLA_U32, 255),
        [ETHTOOL_A_PLCA_BURST_TMR]        = NLA_POLICY_MAX(NLA_U32, 255),
};

static int
ethnl_set_plca(struct ethnl_req_info *req_info, struct genl_info *info)
{
        const struct ethtool_phy_ops *ops;
        struct nlattr **tb = info->attrs;
        struct phy_plca_cfg plca_cfg;
        struct phy_device *phydev;
        bool mod = false;
        int ret;

        phydev = ethnl_req_get_phydev(req_info, tb[ETHTOOL_A_PLCA_HEADER],
                                      info->extack);
        // check that the PHY device is available and connected
        if (IS_ERR_OR_NULL(phydev))
                return -EOPNOTSUPP;

        ops = ethtool_phy_ops;
        if (!ops || !ops->set_plca_cfg)
                return -EOPNOTSUPP;

        memset(&plca_cfg, 0xff, sizeof(plca_cfg));
        plca_update_sint(&plca_cfg.enabled, tb, ETHTOOL_A_PLCA_ENABLED, &mod);
        plca_update_sint(&plca_cfg.node_id, tb, ETHTOOL_A_PLCA_NODE_ID, &mod);
        plca_update_sint(&plca_cfg.node_cnt, tb, ETHTOOL_A_PLCA_NODE_CNT, &mod);
        plca_update_sint(&plca_cfg.to_tmr, tb, ETHTOOL_A_PLCA_TO_TMR, &mod);
        plca_update_sint(&plca_cfg.burst_cnt, tb, ETHTOOL_A_PLCA_BURST_CNT,
                         &mod);
        plca_update_sint(&plca_cfg.burst_tmr, tb, ETHTOOL_A_PLCA_BURST_TMR,
                         &mod);
        if (!mod)
                return 0;

        ret = ops->set_plca_cfg(phydev, &plca_cfg, info->extack);
        return ret < 0 ? ret : 1;
}

const struct ethnl_request_ops ethnl_plca_cfg_request_ops = {
        .request_cmd                = ETHTOOL_MSG_PLCA_GET_CFG,
        .reply_cmd                = ETHTOOL_MSG_PLCA_GET_CFG_REPLY,
        .hdr_attr                = ETHTOOL_A_PLCA_HEADER,
        .req_info_size                = sizeof(struct plca_req_info),
        .reply_data_size        = sizeof(struct plca_reply_data),

        .prepare_data                = plca_get_cfg_prepare_data,
        .reply_size                = plca_get_cfg_reply_size,
        .fill_reply                = plca_get_cfg_fill_reply,

        .set                        = ethnl_set_plca,
        .set_ntf_cmd                = ETHTOOL_MSG_PLCA_NTF,
};

// PLCA get status message -------------------------------------------------- //

const struct nla_policy ethnl_plca_get_status_policy[] = {
        [ETHTOOL_A_PLCA_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy_phy),
};

static int plca_get_status_prepare_data(const struct ethnl_req_info *req_base,
                                        struct ethnl_reply_data *reply_base,
                                        const struct genl_info *info)
{
        struct plca_reply_data *data = PLCA_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        const struct ethtool_phy_ops *ops;
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        int ret;

        phydev = ethnl_req_get_phydev(req_base, tb[ETHTOOL_A_PLCA_HEADER],
                                      info->extack);
        // check that the PHY device is available and connected
        if (IS_ERR_OR_NULL(phydev)) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        // note: rtnl_lock is held already by ethnl_default_doit
        ops = ethtool_phy_ops;
        if (!ops || !ops->get_plca_status) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                goto out;

        memset(&data->plca_st, 0xff,
               sizeof_field(struct plca_reply_data, plca_st));

        ret = ops->get_plca_status(phydev, &data->plca_st);
        ethnl_ops_complete(dev);
out:
        return ret;
}

static int plca_get_status_reply_size(const struct ethnl_req_info *req_base,
                                      const struct ethnl_reply_data *reply_base)
{
        return nla_total_size(sizeof(u8));        /* _STATUS */
}

static int plca_get_status_fill_reply(struct sk_buff *skb,
                                      const struct ethnl_req_info *req_base,
                                      const struct ethnl_reply_data *reply_base)
{
        const struct plca_reply_data *data = PLCA_REPDATA(reply_base);
        const u8 status = data->plca_st.pst;

        if (nla_put_u8(skb, ETHTOOL_A_PLCA_STATUS, !!status))
                return -EMSGSIZE;

        return 0;
};

const struct ethnl_request_ops ethnl_plca_status_request_ops = {
        .request_cmd                = ETHTOOL_MSG_PLCA_GET_STATUS,
        .reply_cmd                = ETHTOOL_MSG_PLCA_GET_STATUS_REPLY,
        .hdr_attr                = ETHTOOL_A_PLCA_HEADER,
        .req_info_size                = sizeof(struct plca_req_info),
        .reply_data_size        = sizeof(struct plca_reply_data),

        .prepare_data                = plca_get_status_prepare_data,
        .reply_size                = plca_get_status_reply_size,
        .fill_reply                = plca_get_status_fill_reply,
};

































































































































































































































































































































































































































   98 













  362 












































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
/*
 * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
 * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
 * Copyright (c) 2009-2010, Code Aurora Forum.
 * All rights reserved.
 *
 * Author: Rickard E. (Rik) Faith <faith@valinux.com>
 * Author: Gareth Hughes <gareth@valinux.com>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */

#ifndef _DRM_FILE_H_
#define _DRM_FILE_H_

#include <linux/types.h>
#include <linux/completion.h>
#include <linux/idr.h>

#include <uapi/drm/drm.h>

#include <drm/drm_prime.h>

struct dma_fence;
struct drm_file;
struct drm_device;
struct drm_printer;
struct device;
struct file;

extern struct xarray drm_minors_xa;

/*
 * FIXME: Not sure we want to have drm_minor here in the end, but to avoid
 * header include loops we need it here for now.
 */

/* Note that the values of this enum are ABI (it determines
 * /dev/dri/renderD* numbers).
 *
 * Setting DRM_MINOR_ACCEL to 32 gives enough space for more drm minors to
 * be implemented before we hit any future
 */
enum drm_minor_type {
        DRM_MINOR_PRIMARY = 0,
        DRM_MINOR_CONTROL = 1,
        DRM_MINOR_RENDER = 2,
        DRM_MINOR_ACCEL = 32,
};

/**
 * struct drm_minor - DRM device minor structure
 *
 * This structure represents a DRM minor number for device nodes in /dev.
 * Entirely opaque to drivers and should never be inspected directly by drivers.
 * Drivers instead should only interact with &struct drm_file and of course
 * &struct drm_device, which is also where driver-private data and resources can
 * be attached to.
 */
struct drm_minor {
        /* private: */
        int index;                        /* Minor device number */
        int type;                       /* Control or render or accel */
        struct device *kdev;                /* Linux device */
        struct drm_device *dev;

        struct dentry *debugfs_symlink;
        struct dentry *debugfs_root;
};

/**
 * struct drm_pending_event - Event queued up for userspace to read
 *
 * This represents a DRM event. Drivers can use this as a generic completion
 * mechanism, which supports kernel-internal &struct completion, &struct dma_fence
 * and also the DRM-specific &struct drm_event delivery mechanism.
 */
struct drm_pending_event {
        /**
         * @completion:
         *
         * Optional pointer to a kernel internal completion signalled when
         * drm_send_event() is called, useful to internally synchronize with
         * nonblocking operations.
         */
        struct completion *completion;

        /**
         * @completion_release:
         *
         * Optional callback currently only used by the atomic modeset helpers
         * to clean up the reference count for the structure @completion is
         * stored in.
         */
        void (*completion_release)(struct completion *completion);

        /**
         * @event:
         *
         * Pointer to the actual event that should be sent to userspace to be
         * read using drm_read(). Can be optional, since nowadays events are
         * also used to signal kernel internal threads with @completion or DMA
         * transactions using @fence.
         */
        struct drm_event *event;

        /**
         * @fence:
         *
         * Optional DMA fence to unblock other hardware transactions which
         * depend upon the nonblocking DRM operation this event represents.
         */
        struct dma_fence *fence;

        /**
         * @file_priv:
         *
         * &struct drm_file where @event should be delivered to. Only set when
         * @event is set.
         */
        struct drm_file *file_priv;

        /**
         * @link:
         *
         * Double-linked list to keep track of this event. Can be used by the
         * driver up to the point when it calls drm_send_event(), after that
         * this list entry is owned by the core for its own book-keeping.
         */
        struct list_head link;

        /**
         * @pending_link:
         *
         * Entry on &drm_file.pending_event_list, to keep track of all pending
         * events for @file_priv, to allow correct unwinding of them when
         * userspace closes the file before the event is delivered.
         */
        struct list_head pending_link;
};

/**
 * struct drm_file - DRM file private data
 *
 * This structure tracks DRM state per open file descriptor.
 */
struct drm_file {
        /**
         * @authenticated:
         *
         * Whether the client is allowed to submit rendering, which for legacy
         * nodes means it must be authenticated.
         *
         * See also the :ref:`section on primary nodes and authentication
         * <drm_primary_node>`.
         */
        bool authenticated;

        /**
         * @stereo_allowed:
         *
         * True when the client has asked us to expose stereo 3D mode flags.
         */
        bool stereo_allowed;

        /**
         * @universal_planes:
         *
         * True if client understands CRTC primary planes and cursor planes
         * in the plane list. Automatically set when @atomic is set.
         */
        bool universal_planes;

        /** @atomic: True if client understands atomic properties. */
        bool atomic;

        /**
         * @aspect_ratio_allowed:
         *
         * True, if client can handle picture aspect ratios, and has requested
         * to pass this information along with the mode.
         */
        bool aspect_ratio_allowed;

        /**
         * @writeback_connectors:
         *
         * True if client understands writeback connectors
         */
        bool writeback_connectors;

        /**
         * @was_master:
         *
         * This client has or had, master capability. Protected by struct
         * &drm_device.master_mutex.
         *
         * This is used to ensure that CAP_SYS_ADMIN is not enforced, if the
         * client is or was master in the past.
         */
        bool was_master;

        /**
         * @is_master:
         *
         * This client is the creator of @master. Protected by struct
         * &drm_device.master_mutex.
         *
         * See also the :ref:`section on primary nodes and authentication
         * <drm_primary_node>`.
         */
        bool is_master;

        /**
         * @supports_virtualized_cursor_plane:
         *
         * This client is capable of handling the cursor plane with the
         * restrictions imposed on it by the virtualized drivers.
         *
         * This implies that the cursor plane has to behave like a cursor
         * i.e. track cursor movement. It also requires setting of the
         * hotspot properties by the client on the cursor plane.
         */
        bool supports_virtualized_cursor_plane;

        /**
         * @master:
         *
         * Master this node is currently associated with. Protected by struct
         * &drm_device.master_mutex, and serialized by @master_lookup_lock.
         *
         * Only relevant if drm_is_primary_client() returns true. Note that
         * this only matches &drm_device.master if the master is the currently
         * active one.
         *
         * To update @master, both &drm_device.master_mutex and
         * @master_lookup_lock need to be held, therefore holding either of
         * them is safe and enough for the read side.
         *
         * When dereferencing this pointer, either hold struct
         * &drm_device.master_mutex for the duration of the pointer's use, or
         * use drm_file_get_master() if struct &drm_device.master_mutex is not
         * currently held and there is no other need to hold it. This prevents
         * @master from being freed during use.
         *
         * See also @authentication and @is_master and the :ref:`section on
         * primary nodes and authentication <drm_primary_node>`.
         */
        struct drm_master *master;

        /** @master_lookup_lock: Serializes @master. */
        spinlock_t master_lookup_lock;

        /**
         * @pid: Process that is using this file.
         *
         * Must only be dereferenced under a rcu_read_lock or equivalent.
         *
         * Updates are guarded with dev->filelist_mutex and reference must be
         * dropped after a RCU grace period to accommodate lockless readers.
         */
        struct pid __rcu *pid;

        /** @client_id: A unique id for fdinfo */
        u64 client_id;

        /** @magic: Authentication magic, see @authenticated. */
        drm_magic_t magic;

        /**
         * @lhead:
         *
         * List of all open files of a DRM device, linked into
         * &drm_device.filelist. Protected by &drm_device.filelist_mutex.
         */
        struct list_head lhead;

        /** @minor: &struct drm_minor for this file. */
        struct drm_minor *minor;

        /**
         * @object_idr:
         *
         * Mapping of mm object handles to object pointers. Used by the GEM
         * subsystem. Protected by @table_lock.
         */
        struct idr object_idr;

        /** @table_lock: Protects @object_idr. */
        spinlock_t table_lock;

        /** @syncobj_idr: Mapping of sync object handles to object pointers. */
        struct idr syncobj_idr;
        /** @syncobj_table_lock: Protects @syncobj_idr. */
        spinlock_t syncobj_table_lock;

        /** @filp: Pointer to the core file structure. */
        struct file *filp;

        /**
         * @driver_priv:
         *
         * Optional pointer for driver private data. Can be allocated in
         * &drm_driver.open and should be freed in &drm_driver.postclose.
         */
        void *driver_priv;

        /**
         * @fbs:
         *
         * List of &struct drm_framebuffer associated with this file, using the
         * &drm_framebuffer.filp_head entry.
         *
         * Protected by @fbs_lock. Note that the @fbs list holds a reference on
         * the framebuffer object to prevent it from untimely disappearing.
         */
        struct list_head fbs;

        /** @fbs_lock: Protects @fbs. */
        struct mutex fbs_lock;

        /**
         * @blobs:
         *
         * User-created blob properties; this retains a reference on the
         * property.
         *
         * Protected by @drm_mode_config.blob_lock;
         */
        struct list_head blobs;

        /** @event_wait: Waitqueue for new events added to @event_list. */
        wait_queue_head_t event_wait;

        /**
         * @pending_event_list:
         *
         * List of pending &struct drm_pending_event, used to clean up pending
         * events in case this file gets closed before the event is signalled.
         * Uses the &drm_pending_event.pending_link entry.
         *
         * Protect by &drm_device.event_lock.
         */
        struct list_head pending_event_list;

        /**
         * @event_list:
         *
         * List of &struct drm_pending_event, ready for delivery to userspace
         * through drm_read(). Uses the &drm_pending_event.link entry.
         *
         * Protect by &drm_device.event_lock.
         */
        struct list_head event_list;

        /**
         * @event_space:
         *
         * Available event space to prevent userspace from
         * exhausting kernel memory. Currently limited to the fairly arbitrary
         * value of 4KB.
         */
        int event_space;

        /** @event_read_lock: Serializes drm_read(). */
        struct mutex event_read_lock;

        /**
         * @prime:
         *
         * Per-file buffer caches used by the PRIME buffer sharing code.
         */
        struct drm_prime_file_private prime;

        /**
         * @client_name:
         *
         * Userspace-provided name; useful for accounting and debugging.
         */
        const char *client_name;

        /**
         * @client_name_lock: Protects @client_name.
         */
        struct mutex client_name_lock;
};

/**
 * drm_is_primary_client - is this an open file of the primary node
 * @file_priv: DRM file
 *
 * Returns true if this is an open file of the primary node, i.e.
 * &drm_file.minor of @file_priv is a primary minor.
 *
 * See also the :ref:`section on primary nodes and authentication
 * <drm_primary_node>`.
 */
static inline bool drm_is_primary_client(const struct drm_file *file_priv)
{
        return file_priv->minor->type == DRM_MINOR_PRIMARY;
}

/**
 * drm_is_render_client - is this an open file of the render node
 * @file_priv: DRM file
 *
 * Returns true if this is an open file of the render node, i.e.
 * &drm_file.minor of @file_priv is a render minor.
 *
 * See also the :ref:`section on render nodes <drm_render_node>`.
 */
static inline bool drm_is_render_client(const struct drm_file *file_priv)
{
        return file_priv->minor->type == DRM_MINOR_RENDER;
}

/**
 * drm_is_accel_client - is this an open file of the compute acceleration node
 * @file_priv: DRM file
 *
 * Returns true if this is an open file of the compute acceleration node, i.e.
 * &drm_file.minor of @file_priv is a accel minor.
 *
 * See also :doc:`Introduction to compute accelerators subsystem
 * </accel/introduction>`.
 */
static inline bool drm_is_accel_client(const struct drm_file *file_priv)
{
        return file_priv->minor->type == DRM_MINOR_ACCEL;
}

void drm_file_update_pid(struct drm_file *);

struct drm_minor *drm_minor_acquire(struct xarray *minors_xa, unsigned int minor_id);
void drm_minor_release(struct drm_minor *minor);

int drm_open(struct inode *inode, struct file *filp);
int drm_open_helper(struct file *filp, struct drm_minor *minor);
ssize_t drm_read(struct file *filp, char __user *buffer,
                 size_t count, loff_t *offset);
int drm_release(struct inode *inode, struct file *filp);
int drm_release_noglobal(struct inode *inode, struct file *filp);
__poll_t drm_poll(struct file *filp, struct poll_table_struct *wait);
int drm_event_reserve_init_locked(struct drm_device *dev,
                                  struct drm_file *file_priv,
                                  struct drm_pending_event *p,
                                  struct drm_event *e);
int drm_event_reserve_init(struct drm_device *dev,
                           struct drm_file *file_priv,
                           struct drm_pending_event *p,
                           struct drm_event *e);
void drm_event_cancel_free(struct drm_device *dev,
                           struct drm_pending_event *p);
void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e);
void drm_send_event(struct drm_device *dev, struct drm_pending_event *e);
void drm_send_event_timestamp_locked(struct drm_device *dev,
                                     struct drm_pending_event *e,
                                     ktime_t timestamp);

/**
 * struct drm_memory_stats - GEM object stats associated
 * @shared: Total size of GEM objects shared between processes
 * @private: Total size of GEM objects
 * @resident: Total size of GEM objects backing pages
 * @purgeable: Total size of GEM objects that can be purged (resident and not active)
 * @active: Total size of GEM objects active on one or more engines
 *
 * Used by drm_print_memory_stats()
 */
struct drm_memory_stats {
        u64 shared;
        u64 private;
        u64 resident;
        u64 purgeable;
        u64 active;
};

enum drm_gem_object_status;

void drm_print_memory_stats(struct drm_printer *p,
                            const struct drm_memory_stats *stats,
                            enum drm_gem_object_status supported_status,
                            const char *region);

void drm_show_memory_stats(struct drm_printer *p, struct drm_file *file);
void drm_show_fdinfo(struct seq_file *m, struct file *f);

struct file *mock_drm_getfile(struct drm_minor *minor, unsigned int flags);

#endif /* _DRM_FILE_H_ */









































































































































































































































































































































































































































































































   36 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
// SPDX-License-Identifier: GPL-2.0

#include "bcachefs.h"
#include "alloc_background.h"
#include "alloc_foreground.h"
#include "btree_iter.h"
#include "btree_update.h"
#include "btree_write_buffer.h"
#include "buckets.h"
#include "clock.h"
#include "compress.h"
#include "disk_groups.h"
#include "errcode.h"
#include "error.h"
#include "inode.h"
#include "io_write.h"
#include "move.h"
#include "rebalance.h"
#include "subvolume.h"
#include "super-io.h"
#include "trace.h"

#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/sched/cputime.h>

#define REBALANCE_WORK_SCAN_OFFSET        (U64_MAX - 1)

static const char * const bch2_rebalance_state_strs[] = {
#define x(t) #t,
        BCH_REBALANCE_STATES()
        NULL
#undef x
};

static int __bch2_set_rebalance_needs_scan(struct btree_trans *trans, u64 inum)
{
        struct btree_iter iter;
        struct bkey_s_c k;
        struct bkey_i_cookie *cookie;
        u64 v;
        int ret;

        bch2_trans_iter_init(trans, &iter, BTREE_ID_rebalance_work,
                             SPOS(inum, REBALANCE_WORK_SCAN_OFFSET, U32_MAX),
                             BTREE_ITER_intent);
        k = bch2_btree_iter_peek_slot(&iter);
        ret = bkey_err(k);
        if (ret)
                goto err;

        v = k.k->type == KEY_TYPE_cookie
                ? le64_to_cpu(bkey_s_c_to_cookie(k).v->cookie)
                : 0;

        cookie = bch2_trans_kmalloc(trans, sizeof(*cookie));
        ret = PTR_ERR_OR_ZERO(cookie);
        if (ret)
                goto err;

        bkey_cookie_init(&cookie->k_i);
        cookie->k.p = iter.pos;
        cookie->v.cookie = cpu_to_le64(v + 1);

        ret = bch2_trans_update(trans, &iter, &cookie->k_i, 0);
err:
        bch2_trans_iter_exit(trans, &iter);
        return ret;
}

int bch2_set_rebalance_needs_scan(struct bch_fs *c, u64 inum)
{
        int ret = bch2_trans_commit_do(c, NULL, NULL,
                                       BCH_TRANS_COMMIT_no_enospc|
                                       BCH_TRANS_COMMIT_lazy_rw,
                            __bch2_set_rebalance_needs_scan(trans, inum));
        rebalance_wakeup(c);
        return ret;
}

int bch2_set_fs_needs_rebalance(struct bch_fs *c)
{
        return bch2_set_rebalance_needs_scan(c, 0);
}

static int bch2_clear_rebalance_needs_scan(struct btree_trans *trans, u64 inum, u64 cookie)
{
        struct btree_iter iter;
        struct bkey_s_c k;
        u64 v;
        int ret;

        bch2_trans_iter_init(trans, &iter, BTREE_ID_rebalance_work,
                             SPOS(inum, REBALANCE_WORK_SCAN_OFFSET, U32_MAX),
                             BTREE_ITER_intent);
        k = bch2_btree_iter_peek_slot(&iter);
        ret = bkey_err(k);
        if (ret)
                goto err;

        v = k.k->type == KEY_TYPE_cookie
                ? le64_to_cpu(bkey_s_c_to_cookie(k).v->cookie)
                : 0;

        if (v == cookie)
                ret = bch2_btree_delete_at(trans, &iter, 0);
err:
        bch2_trans_iter_exit(trans, &iter);
        return ret;
}

static struct bkey_s_c next_rebalance_entry(struct btree_trans *trans,
                                            struct btree_iter *work_iter)
{
        return !kthread_should_stop()
                ? bch2_btree_iter_peek(work_iter)
                : bkey_s_c_null;
}

static int bch2_bkey_clear_needs_rebalance(struct btree_trans *trans,
                                           struct btree_iter *iter,
                                           struct bkey_s_c k)
{
        struct bkey_i *n = bch2_bkey_make_mut(trans, iter, &k, 0);
        int ret = PTR_ERR_OR_ZERO(n);
        if (ret)
                return ret;

        extent_entry_drop(bkey_i_to_s(n),
                          (void *) bch2_bkey_rebalance_opts(bkey_i_to_s_c(n)));
        return bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc);
}

static struct bkey_s_c next_rebalance_extent(struct btree_trans *trans,
                        struct bpos work_pos,
                        struct btree_iter *extent_iter,
                        struct data_update_opts *data_opts)
{
        struct bch_fs *c = trans->c;
        struct bkey_s_c k;

        bch2_trans_iter_exit(trans, extent_iter);
        bch2_trans_iter_init(trans, extent_iter,
                             work_pos.inode ? BTREE_ID_extents : BTREE_ID_reflink,
                             work_pos,
                             BTREE_ITER_all_snapshots);
        k = bch2_btree_iter_peek_slot(extent_iter);
        if (bkey_err(k))
                return k;

        const struct bch_extent_rebalance *r = k.k ? bch2_bkey_rebalance_opts(k) : NULL;
        if (!r) {
                /* raced due to btree write buffer, nothing to do */
                return bkey_s_c_null;
        }

        memset(data_opts, 0, sizeof(*data_opts));

        data_opts->rewrite_ptrs                =
                bch2_bkey_ptrs_need_rebalance(c, k, r->target, r->compression);
        data_opts->target                = r->target;
        data_opts->write_flags                |= BCH_WRITE_ONLY_SPECIFIED_DEVS;

        if (!data_opts->rewrite_ptrs) {
                /*
                 * device we would want to write to offline? devices in target
                 * changed?
                 *
                 * We'll now need a full scan before this extent is picked up
                 * again:
                 */
                int ret = bch2_bkey_clear_needs_rebalance(trans, extent_iter, k);
                if (ret)
                        return bkey_s_c_err(ret);
                return bkey_s_c_null;
        }

        if (trace_rebalance_extent_enabled()) {
                struct printbuf buf = PRINTBUF;

                prt_str(&buf, "target=");
                bch2_target_to_text(&buf, c, r->target);
                prt_str(&buf, " compression=");
                bch2_compression_opt_to_text(&buf, r->compression);
                prt_str(&buf, " ");
                bch2_bkey_val_to_text(&buf, c, k);

                trace_rebalance_extent(c, buf.buf);
                printbuf_exit(&buf);
        }

        return k;
}

noinline_for_stack
static int do_rebalance_extent(struct moving_context *ctxt,
                               struct bpos work_pos,
                               struct btree_iter *extent_iter)
{
        struct btree_trans *trans = ctxt->trans;
        struct bch_fs *c = trans->c;
        struct bch_fs_rebalance *r = &trans->c->rebalance;
        struct data_update_opts data_opts;
        struct bch_io_opts io_opts;
        struct bkey_s_c k;
        struct bkey_buf sk;
        int ret;

        ctxt->stats = &r->work_stats;
        r->state = BCH_REBALANCE_working;

        bch2_bkey_buf_init(&sk);

        ret = bkey_err(k = next_rebalance_extent(trans, work_pos,
                                                 extent_iter, &data_opts));
        if (ret || !k.k)
                goto out;

        ret = bch2_move_get_io_opts_one(trans, &io_opts, k);
        if (ret)
                goto out;

        atomic64_add(k.k->size, &ctxt->stats->sectors_seen);

        /*
         * The iterator gets unlocked by __bch2_read_extent - need to
         * save a copy of @k elsewhere:
         */
        bch2_bkey_buf_reassemble(&sk, c, k);
        k = bkey_i_to_s_c(sk.k);

        ret = bch2_move_extent(ctxt, NULL, extent_iter, k, io_opts, data_opts);
        if (ret) {
                if (bch2_err_matches(ret, ENOMEM)) {
                        /* memory allocation failure, wait for some IO to finish */
                        bch2_move_ctxt_wait_for_io(ctxt);
                        ret = -BCH_ERR_transaction_restart_nested;
                }

                if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
                        goto out;

                /* skip it and continue, XXX signal failure */
                ret = 0;
        }
out:
        bch2_bkey_buf_exit(&sk, c);
        return ret;
}

static bool rebalance_pred(struct bch_fs *c, void *arg,
                           struct bkey_s_c k,
                           struct bch_io_opts *io_opts,
                           struct data_update_opts *data_opts)
{
        unsigned target, compression;

        if (k.k->p.inode) {
                target                = io_opts->background_target;
                compression        = background_compression(*io_opts);
        } else {
                const struct bch_extent_rebalance *r = bch2_bkey_rebalance_opts(k);

                target                = r ? r->target : io_opts->background_target;
                compression        = r ? r->compression : background_compression(*io_opts);
        }

        data_opts->rewrite_ptrs                = bch2_bkey_ptrs_need_rebalance(c, k, target, compression);
        data_opts->target                = target;
        data_opts->write_flags                |= BCH_WRITE_ONLY_SPECIFIED_DEVS;
        return data_opts->rewrite_ptrs != 0;
}

static int do_rebalance_scan(struct moving_context *ctxt, u64 inum, u64 cookie)
{
        struct btree_trans *trans = ctxt->trans;
        struct bch_fs_rebalance *r = &trans->c->rebalance;
        int ret;

        bch2_move_stats_init(&r->scan_stats, "rebalance_scan");
        ctxt->stats = &r->scan_stats;

        if (!inum) {
                r->scan_start        = BBPOS_MIN;
                r->scan_end        = BBPOS_MAX;
        } else {
                r->scan_start        = BBPOS(BTREE_ID_extents, POS(inum, 0));
                r->scan_end        = BBPOS(BTREE_ID_extents, POS(inum, U64_MAX));
        }

        r->state = BCH_REBALANCE_scanning;

        ret = __bch2_move_data(ctxt, r->scan_start, r->scan_end, rebalance_pred, NULL) ?:
                commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
                          bch2_clear_rebalance_needs_scan(trans, inum, cookie));

        bch2_move_stats_exit(&r->scan_stats, trans->c);
        return ret;
}

static void rebalance_wait(struct bch_fs *c)
{
        struct bch_fs_rebalance *r = &c->rebalance;
        struct io_clock *clock = &c->io_clock[WRITE];
        u64 now = atomic64_read(&clock->now);
        u64 min_member_capacity = bch2_min_rw_member_capacity(c);

        if (min_member_capacity == U64_MAX)
                min_member_capacity = 128 * 2048;

        r->wait_iotime_end                = now + (min_member_capacity >> 6);

        if (r->state != BCH_REBALANCE_waiting) {
                r->wait_iotime_start        = now;
                r->wait_wallclock_start        = ktime_get_real_ns();
                r->state                = BCH_REBALANCE_waiting;
        }

        bch2_kthread_io_clock_wait(clock, r->wait_iotime_end, MAX_SCHEDULE_TIMEOUT);
}

static int do_rebalance(struct moving_context *ctxt)
{
        struct btree_trans *trans = ctxt->trans;
        struct bch_fs *c = trans->c;
        struct bch_fs_rebalance *r = &c->rebalance;
        struct btree_iter rebalance_work_iter, extent_iter = { NULL };
        struct bkey_s_c k;
        int ret = 0;

        bch2_trans_begin(trans);

        bch2_move_stats_init(&r->work_stats, "rebalance_work");
        bch2_move_stats_init(&r->scan_stats, "rebalance_scan");

        bch2_trans_iter_init(trans, &rebalance_work_iter,
                             BTREE_ID_rebalance_work, POS_MIN,
                             BTREE_ITER_all_snapshots);

        while (!bch2_move_ratelimit(ctxt)) {
                if (!r->enabled) {
                        bch2_moving_ctxt_flush_all(ctxt);
                        kthread_wait_freezable(r->enabled ||
                                               kthread_should_stop());
                }

                if (kthread_should_stop())
                        break;

                bch2_trans_begin(trans);

                ret = bkey_err(k = next_rebalance_entry(trans, &rebalance_work_iter));
                if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
                        continue;
                if (ret || !k.k)
                        break;

                ret = k.k->type == KEY_TYPE_cookie
                        ? do_rebalance_scan(ctxt, k.k->p.inode,
                                            le64_to_cpu(bkey_s_c_to_cookie(k).v->cookie))
                        : do_rebalance_extent(ctxt, k.k->p, &extent_iter);

                if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
                        continue;
                if (ret)
                        break;

                bch2_btree_iter_advance(&rebalance_work_iter);
        }

        bch2_trans_iter_exit(trans, &extent_iter);
        bch2_trans_iter_exit(trans, &rebalance_work_iter);
        bch2_move_stats_exit(&r->scan_stats, c);

        if (!ret &&
            !kthread_should_stop() &&
            !atomic64_read(&r->work_stats.sectors_seen) &&
            !atomic64_read(&r->scan_stats.sectors_seen)) {
                bch2_moving_ctxt_flush_all(ctxt);
                bch2_trans_unlock_long(trans);
                rebalance_wait(c);
        }

        if (!bch2_err_matches(ret, EROFS))
                bch_err_fn(c, ret);
        return ret;
}

static int bch2_rebalance_thread(void *arg)
{
        struct bch_fs *c = arg;
        struct bch_fs_rebalance *r = &c->rebalance;
        struct moving_context ctxt;

        set_freezable();

        bch2_moving_ctxt_init(&ctxt, c, NULL, &r->work_stats,
                              writepoint_ptr(&c->rebalance_write_point),
                              true);

        while (!kthread_should_stop() && !do_rebalance(&ctxt))
                ;

        bch2_moving_ctxt_exit(&ctxt);

        return 0;
}

void bch2_rebalance_status_to_text(struct printbuf *out, struct bch_fs *c)
{
        struct bch_fs_rebalance *r = &c->rebalance;

        prt_str(out, bch2_rebalance_state_strs[r->state]);
        prt_newline(out);
        printbuf_indent_add(out, 2);

        switch (r->state) {
        case BCH_REBALANCE_waiting: {
                u64 now = atomic64_read(&c->io_clock[WRITE].now);

                prt_str(out, "io wait duration:  ");
                bch2_prt_human_readable_s64(out, (r->wait_iotime_end - r->wait_iotime_start) << 9);
                prt_newline(out);

                prt_str(out, "io wait remaining: ");
                bch2_prt_human_readable_s64(out, (r->wait_iotime_end - now) << 9);
                prt_newline(out);

                prt_str(out, "duration waited:   ");
                bch2_pr_time_units(out, ktime_get_real_ns() - r->wait_wallclock_start);
                prt_newline(out);
                break;
        }
        case BCH_REBALANCE_working:
                bch2_move_stats_to_text(out, &r->work_stats);
                break;
        case BCH_REBALANCE_scanning:
                bch2_move_stats_to_text(out, &r->scan_stats);
                break;
        }
        prt_newline(out);
        printbuf_indent_sub(out, 2);
}

void bch2_rebalance_stop(struct bch_fs *c)
{
        struct task_struct *p;

        c->rebalance.pd.rate.rate = UINT_MAX;
        bch2_ratelimit_reset(&c->rebalance.pd.rate);

        p = rcu_dereference_protected(c->rebalance.thread, 1);
        c->rebalance.thread = NULL;

        if (p) {
                /* for sychronizing with rebalance_wakeup() */
                synchronize_rcu();

                kthread_stop(p);
                put_task_struct(p);
        }
}

int bch2_rebalance_start(struct bch_fs *c)
{
        struct task_struct *p;
        int ret;

        if (c->rebalance.thread)
                return 0;

        if (c->opts.nochanges)
                return 0;

        p = kthread_create(bch2_rebalance_thread, c, "bch-rebalance/%s", c->name);
        ret = PTR_ERR_OR_ZERO(p);
        bch_err_msg(c, ret, "creating rebalance thread");
        if (ret)
                return ret;

        get_task_struct(p);
        rcu_assign_pointer(c->rebalance.thread, p);
        wake_up_process(p);
        return 0;
}

void bch2_fs_rebalance_init(struct bch_fs *c)
{
        bch2_pd_controller_init(&c->rebalance.pd);
}






























   20 


   21 





   30 

   12 




   18 


   30 








   45 

  277 











   12 



   17 



   14 








    9 



    9 



   17 







    6 
    2 





    4 










    2 

    2 









    1 









    1 
    1 
    1 




































   12 
   12 
    1 






   11 

    1 




   10 
    5 



    9 






   30 







   18 
   18 
    1 






   16 



    1 




   16 



    1 




   15 
    3 




    3 

    2 




    1 











   13 
    2 



   12 

    2 


   10 
    1 





















   29 

   30 

   18 



















   28 






    9 















   19 













   28 





















   12 

   12 


    8 




    8 














    4 











   12 










   38 





    8 
   38 


   30 












   24 





   24 

   12 












    5 




    7 


    6 








   11 







   32 









   30 















   29 

   30 




   29 



   29 



   28 

   27 

   26 


   24 
    1 
    1 





   29 


   32 






    3 






    2 


    1 
    3 







    3 













    2 

    2 


    2 

    1 

    3 








    1 






    1 

    1 


    1 
























    6 



    2 

    6 


    6 



    6 

















    3 


    3 



    3 










    3 


    3 












  260 
   16 

    8 
    7 


  250 















   18 








   18 















   18 











   18 
   13 


   18 





















   29 


   29 



    9 
    7 




   23 
   21 











   27 
    9 





   27 



   29 
   29 













    3 

    3 

    3 















   21 





    1 



    1 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
// SPDX-License-Identifier: GPL-2.0
/*
 * Encryption policy functions for per-file encryption support.
 *
 * Copyright (C) 2015, Google, Inc.
 * Copyright (C) 2015, Motorola Mobility.
 *
 * Originally written by Michael Halcrow, 2015.
 * Modified by Jaegeuk Kim, 2015.
 * Modified by Eric Biggers, 2019 for v2 policy support.
 */

#include <linux/fs_context.h>
#include <linux/random.h>
#include <linux/seq_file.h>
#include <linux/string.h>
#include <linux/mount.h>
#include "fscrypt_private.h"

/**
 * fscrypt_policies_equal() - check whether two encryption policies are the same
 * @policy1: the first policy
 * @policy2: the second policy
 *
 * Return: %true if equal, else %false
 */
bool fscrypt_policies_equal(const union fscrypt_policy *policy1,
                            const union fscrypt_policy *policy2)
{
        if (policy1->version != policy2->version)
                return false;

        return !memcmp(policy1, policy2, fscrypt_policy_size(policy1));
}

int fscrypt_policy_to_key_spec(const union fscrypt_policy *policy,
                               struct fscrypt_key_specifier *key_spec)
{
        switch (policy->version) {
        case FSCRYPT_POLICY_V1:
                key_spec->type = FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR;
                memcpy(key_spec->u.descriptor, policy->v1.master_key_descriptor,
                       FSCRYPT_KEY_DESCRIPTOR_SIZE);
                return 0;
        case FSCRYPT_POLICY_V2:
                key_spec->type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER;
                memcpy(key_spec->u.identifier, policy->v2.master_key_identifier,
                       FSCRYPT_KEY_IDENTIFIER_SIZE);
                return 0;
        default:
                WARN_ON_ONCE(1);
                return -EINVAL;
        }
}

const union fscrypt_policy *fscrypt_get_dummy_policy(struct super_block *sb)
{
        if (!sb->s_cop->get_dummy_policy)
                return NULL;
        return sb->s_cop->get_dummy_policy(sb);
}

/*
 * Return %true if the given combination of encryption modes is supported for v1
 * (and later) encryption policies.
 *
 * Do *not* add anything new here, since v1 encryption policies are deprecated.
 * New combinations of modes should go in fscrypt_valid_enc_modes_v2() only.
 */
static bool fscrypt_valid_enc_modes_v1(u32 contents_mode, u32 filenames_mode)
{
        if (contents_mode == FSCRYPT_MODE_AES_256_XTS &&
            filenames_mode == FSCRYPT_MODE_AES_256_CTS)
                return true;

        if (contents_mode == FSCRYPT_MODE_AES_128_CBC &&
            filenames_mode == FSCRYPT_MODE_AES_128_CTS)
                return true;

        if (contents_mode == FSCRYPT_MODE_ADIANTUM &&
            filenames_mode == FSCRYPT_MODE_ADIANTUM)
                return true;

        return false;
}

static bool fscrypt_valid_enc_modes_v2(u32 contents_mode, u32 filenames_mode)
{
        if (contents_mode == FSCRYPT_MODE_AES_256_XTS &&
            filenames_mode == FSCRYPT_MODE_AES_256_HCTR2)
                return true;

        if (contents_mode == FSCRYPT_MODE_SM4_XTS &&
            filenames_mode == FSCRYPT_MODE_SM4_CTS)
                return true;

        return fscrypt_valid_enc_modes_v1(contents_mode, filenames_mode);
}

static bool supported_direct_key_modes(const struct inode *inode,
                                       u32 contents_mode, u32 filenames_mode)
{
        const struct fscrypt_mode *mode;

        if (contents_mode != filenames_mode) {
                fscrypt_warn(inode,
                             "Direct key flag not allowed with different contents and filenames modes");
                return false;
        }
        mode = &fscrypt_modes[contents_mode];

        if (mode->ivsize < offsetofend(union fscrypt_iv, nonce)) {
                fscrypt_warn(inode, "Direct key flag not allowed with %s",
                             mode->friendly_name);
                return false;
        }
        return true;
}

static bool supported_iv_ino_lblk_policy(const struct fscrypt_policy_v2 *policy,
                                         const struct inode *inode)
{
        const char *type = (policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64)
                                ? "IV_INO_LBLK_64" : "IV_INO_LBLK_32";
        struct super_block *sb = inode->i_sb;

        /*
         * IV_INO_LBLK_* exist only because of hardware limitations, and
         * currently the only known use case for them involves AES-256-XTS.
         * That's also all we test currently.  For these reasons, for now only
         * allow AES-256-XTS here.  This can be relaxed later if a use case for
         * IV_INO_LBLK_* with other encryption modes arises.
         */
        if (policy->contents_encryption_mode != FSCRYPT_MODE_AES_256_XTS) {
                fscrypt_warn(inode,
                             "Can't use %s policy with contents mode other than AES-256-XTS",
                             type);
                return false;
        }

        /*
         * It's unsafe to include inode numbers in the IVs if the filesystem can
         * potentially renumber inodes, e.g. via filesystem shrinking.
         */
        if (!sb->s_cop->has_stable_inodes ||
            !sb->s_cop->has_stable_inodes(sb)) {
                fscrypt_warn(inode,
                             "Can't use %s policy on filesystem '%s' because it doesn't have stable inode numbers",
                             type, sb->s_id);
                return false;
        }

        /*
         * IV_INO_LBLK_64 and IV_INO_LBLK_32 both require that inode numbers fit
         * in 32 bits.  In principle, IV_INO_LBLK_32 could support longer inode
         * numbers because it hashes the inode number; however, currently the
         * inode number is gotten from inode::i_ino which is 'unsigned long'.
         * So for now the implementation limit is 32 bits.
         */
        if (!sb->s_cop->has_32bit_inodes) {
                fscrypt_warn(inode,
                             "Can't use %s policy on filesystem '%s' because its inode numbers are too long",
                             type, sb->s_id);
                return false;
        }

        /*
         * IV_INO_LBLK_64 and IV_INO_LBLK_32 both require that file data unit
         * indices fit in 32 bits.
         */
        if (fscrypt_max_file_dun_bits(sb,
                        fscrypt_policy_v2_du_bits(policy, inode)) > 32) {
                fscrypt_warn(inode,
                             "Can't use %s policy on filesystem '%s' because its maximum file size is too large",
                             type, sb->s_id);
                return false;
        }
        return true;
}

static bool fscrypt_supported_v1_policy(const struct fscrypt_policy_v1 *policy,
                                        const struct inode *inode)
{
        if (!fscrypt_valid_enc_modes_v1(policy->contents_encryption_mode,
                                     policy->filenames_encryption_mode)) {
                fscrypt_warn(inode,
                             "Unsupported encryption modes (contents %d, filenames %d)",
                             policy->contents_encryption_mode,
                             policy->filenames_encryption_mode);
                return false;
        }

        if (policy->flags & ~(FSCRYPT_POLICY_FLAGS_PAD_MASK |
                              FSCRYPT_POLICY_FLAG_DIRECT_KEY)) {
                fscrypt_warn(inode, "Unsupported encryption flags (0x%02x)",
                             policy->flags);
                return false;
        }

        if ((policy->flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) &&
            !supported_direct_key_modes(inode, policy->contents_encryption_mode,
                                        policy->filenames_encryption_mode))
                return false;

        if (IS_CASEFOLDED(inode)) {
                /* With v1, there's no way to derive dirhash keys. */
                fscrypt_warn(inode,
                             "v1 policies can't be used on casefolded directories");
                return false;
        }

        return true;
}

static bool fscrypt_supported_v2_policy(const struct fscrypt_policy_v2 *policy,
                                        const struct inode *inode)
{
        int count = 0;

        if (!fscrypt_valid_enc_modes_v2(policy->contents_encryption_mode,
                                     policy->filenames_encryption_mode)) {
                fscrypt_warn(inode,
                             "Unsupported encryption modes (contents %d, filenames %d)",
                             policy->contents_encryption_mode,
                             policy->filenames_encryption_mode);
                return false;
        }

        if (policy->flags & ~(FSCRYPT_POLICY_FLAGS_PAD_MASK |
                              FSCRYPT_POLICY_FLAG_DIRECT_KEY |
                              FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 |
                              FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) {
                fscrypt_warn(inode, "Unsupported encryption flags (0x%02x)",
                             policy->flags);
                return false;
        }

        count += !!(policy->flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY);
        count += !!(policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64);
        count += !!(policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32);
        if (count > 1) {
                fscrypt_warn(inode, "Mutually exclusive encryption flags (0x%02x)",
                             policy->flags);
                return false;
        }

        if (policy->log2_data_unit_size) {
                if (!inode->i_sb->s_cop->supports_subblock_data_units) {
                        fscrypt_warn(inode,
                                     "Filesystem does not support configuring crypto data unit size");
                        return false;
                }
                if (policy->log2_data_unit_size > inode->i_blkbits ||
                    policy->log2_data_unit_size < SECTOR_SHIFT /* 9 */) {
                        fscrypt_warn(inode,
                                     "Unsupported log2_data_unit_size in encryption policy: %d",
                                     policy->log2_data_unit_size);
                        return false;
                }
                if (policy->log2_data_unit_size != inode->i_blkbits &&
                    (policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) {
                        /*
                         * Not safe to enable yet, as we need to ensure that DUN
                         * wraparound can only occur on a FS block boundary.
                         */
                        fscrypt_warn(inode,
                                     "Sub-block data units not yet supported with IV_INO_LBLK_32");
                        return false;
                }
        }

        if ((policy->flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) &&
            !supported_direct_key_modes(inode, policy->contents_encryption_mode,
                                        policy->filenames_encryption_mode))
                return false;

        if ((policy->flags & (FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 |
                              FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) &&
            !supported_iv_ino_lblk_policy(policy, inode))
                return false;

        if (memchr_inv(policy->__reserved, 0, sizeof(policy->__reserved))) {
                fscrypt_warn(inode, "Reserved bits set in encryption policy");
                return false;
        }

        return true;
}

/**
 * fscrypt_supported_policy() - check whether an encryption policy is supported
 * @policy_u: the encryption policy
 * @inode: the inode on which the policy will be used
 *
 * Given an encryption policy, check whether all its encryption modes and other
 * settings are supported by this kernel on the given inode.  (But we don't
 * currently don't check for crypto API support here, so attempting to use an
 * algorithm not configured into the crypto API will still fail later.)
 *
 * Return: %true if supported, else %false
 */
bool fscrypt_supported_policy(const union fscrypt_policy *policy_u,
                              const struct inode *inode)
{
        switch (policy_u->version) {
        case FSCRYPT_POLICY_V1:
                return fscrypt_supported_v1_policy(&policy_u->v1, inode);
        case FSCRYPT_POLICY_V2:
                return fscrypt_supported_v2_policy(&policy_u->v2, inode);
        }
        return false;
}

/**
 * fscrypt_new_context() - create a new fscrypt_context
 * @ctx_u: output context
 * @policy_u: input policy
 * @nonce: nonce to use
 *
 * Create an fscrypt_context for an inode that is being assigned the given
 * encryption policy.  @nonce must be a new random nonce.
 *
 * Return: the size of the new context in bytes.
 */
static int fscrypt_new_context(union fscrypt_context *ctx_u,
                               const union fscrypt_policy *policy_u,
                               const u8 nonce[FSCRYPT_FILE_NONCE_SIZE])
{
        memset(ctx_u, 0, sizeof(*ctx_u));

        switch (policy_u->version) {
        case FSCRYPT_POLICY_V1: {
                const struct fscrypt_policy_v1 *policy = &policy_u->v1;
                struct fscrypt_context_v1 *ctx = &ctx_u->v1;

                ctx->version = FSCRYPT_CONTEXT_V1;
                ctx->contents_encryption_mode =
                        policy->contents_encryption_mode;
                ctx->filenames_encryption_mode =
                        policy->filenames_encryption_mode;
                ctx->flags = policy->flags;
                memcpy(ctx->master_key_descriptor,
                       policy->master_key_descriptor,
                       sizeof(ctx->master_key_descriptor));
                memcpy(ctx->nonce, nonce, FSCRYPT_FILE_NONCE_SIZE);
                return sizeof(*ctx);
        }
        case FSCRYPT_POLICY_V2: {
                const struct fscrypt_policy_v2 *policy = &policy_u->v2;
                struct fscrypt_context_v2 *ctx = &ctx_u->v2;

                ctx->version = FSCRYPT_CONTEXT_V2;
                ctx->contents_encryption_mode =
                        policy->contents_encryption_mode;
                ctx->filenames_encryption_mode =
                        policy->filenames_encryption_mode;
                ctx->flags = policy->flags;
                ctx->log2_data_unit_size = policy->log2_data_unit_size;
                memcpy(ctx->master_key_identifier,
                       policy->master_key_identifier,
                       sizeof(ctx->master_key_identifier));
                memcpy(ctx->nonce, nonce, FSCRYPT_FILE_NONCE_SIZE);
                return sizeof(*ctx);
        }
        }
        BUG();
}

/**
 * fscrypt_policy_from_context() - convert an fscrypt_context to
 *                                   an fscrypt_policy
 * @policy_u: output policy
 * @ctx_u: input context
 * @ctx_size: size of input context in bytes
 *
 * Given an fscrypt_context, build the corresponding fscrypt_policy.
 *
 * Return: 0 on success, or -EINVAL if the fscrypt_context has an unrecognized
 * version number or size.
 *
 * This does *not* validate the settings within the policy itself, e.g. the
 * modes, flags, and reserved bits.  Use fscrypt_supported_policy() for that.
 */
int fscrypt_policy_from_context(union fscrypt_policy *policy_u,
                                const union fscrypt_context *ctx_u,
                                int ctx_size)
{
        memset(policy_u, 0, sizeof(*policy_u));

        if (!fscrypt_context_is_valid(ctx_u, ctx_size))
                return -EINVAL;

        switch (ctx_u->version) {
        case FSCRYPT_CONTEXT_V1: {
                const struct fscrypt_context_v1 *ctx = &ctx_u->v1;
                struct fscrypt_policy_v1 *policy = &policy_u->v1;

                policy->version = FSCRYPT_POLICY_V1;
                policy->contents_encryption_mode =
                        ctx->contents_encryption_mode;
                policy->filenames_encryption_mode =
                        ctx->filenames_encryption_mode;
                policy->flags = ctx->flags;
                memcpy(policy->master_key_descriptor,
                       ctx->master_key_descriptor,
                       sizeof(policy->master_key_descriptor));
                return 0;
        }
        case FSCRYPT_CONTEXT_V2: {
                const struct fscrypt_context_v2 *ctx = &ctx_u->v2;
                struct fscrypt_policy_v2 *policy = &policy_u->v2;

                policy->version = FSCRYPT_POLICY_V2;
                policy->contents_encryption_mode =
                        ctx->contents_encryption_mode;
                policy->filenames_encryption_mode =
                        ctx->filenames_encryption_mode;
                policy->flags = ctx->flags;
                policy->log2_data_unit_size = ctx->log2_data_unit_size;
                memcpy(policy->__reserved, ctx->__reserved,
                       sizeof(policy->__reserved));
                memcpy(policy->master_key_identifier,
                       ctx->master_key_identifier,
                       sizeof(policy->master_key_identifier));
                return 0;
        }
        }
        /* unreachable */
        return -EINVAL;
}

/* Retrieve an inode's encryption policy */
static int fscrypt_get_policy(struct inode *inode, union fscrypt_policy *policy)
{
        const struct fscrypt_inode_info *ci;
        union fscrypt_context ctx;
        int ret;

        ci = fscrypt_get_inode_info(inode);
        if (ci) {
                /* key available, use the cached policy */
                *policy = ci->ci_policy;
                return 0;
        }

        if (!IS_ENCRYPTED(inode))
                return -ENODATA;

        ret = inode->i_sb->s_cop->get_context(inode, &ctx, sizeof(ctx));
        if (ret < 0)
                return (ret == -ERANGE) ? -EINVAL : ret;

        return fscrypt_policy_from_context(policy, &ctx, ret);
}

static int set_encryption_policy(struct inode *inode,
                                 const union fscrypt_policy *policy)
{
        u8 nonce[FSCRYPT_FILE_NONCE_SIZE];
        union fscrypt_context ctx;
        int ctxsize;
        int err;

        if (!fscrypt_supported_policy(policy, inode))
                return -EINVAL;

        switch (policy->version) {
        case FSCRYPT_POLICY_V1:
                /*
                 * The original encryption policy version provided no way of
                 * verifying that the correct master key was supplied, which was
                 * insecure in scenarios where multiple users have access to the
                 * same encrypted files (even just read-only access).  The new
                 * encryption policy version fixes this and also implies use of
                 * an improved key derivation function and allows non-root users
                 * to securely remove keys.  So as long as compatibility with
                 * old kernels isn't required, it is recommended to use the new
                 * policy version for all new encrypted directories.
                 */
                pr_warn_once("%s (pid %d) is setting deprecated v1 encryption policy; recommend upgrading to v2.\n",
                             current->comm, current->pid);
                break;
        case FSCRYPT_POLICY_V2:
                err = fscrypt_verify_key_added(inode->i_sb,
                                               policy->v2.master_key_identifier);
                if (err)
                        return err;
                if (policy->v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)
                        pr_warn_once("%s (pid %d) is setting an IV_INO_LBLK_32 encryption policy.  This should only be used if there are certain hardware limitations.\n",
                                     current->comm, current->pid);
                break;
        default:
                WARN_ON_ONCE(1);
                return -EINVAL;
        }

        get_random_bytes(nonce, FSCRYPT_FILE_NONCE_SIZE);
        ctxsize = fscrypt_new_context(&ctx, policy, nonce);

        return inode->i_sb->s_cop->set_context(inode, &ctx, ctxsize, NULL);
}

int fscrypt_ioctl_set_policy(struct file *filp, const void __user *arg)
{
        union fscrypt_policy policy;
        union fscrypt_policy existing_policy;
        struct inode *inode = file_inode(filp);
        u8 version;
        int size;
        int ret;

        if (get_user(policy.version, (const u8 __user *)arg))
                return -EFAULT;

        size = fscrypt_policy_size(&policy);
        if (size <= 0)
                return -EINVAL;

        /*
         * We should just copy the remaining 'size - 1' bytes here, but a
         * bizarre bug in gcc 7 and earlier (fixed by gcc r255731) causes gcc to
         * think that size can be 0 here (despite the check above!) *and* that
         * it's a compile-time constant.  Thus it would think copy_from_user()
         * is passed compile-time constant ULONG_MAX, causing the compile-time
         * buffer overflow check to fail, breaking the build. This only occurred
         * when building an i386 kernel with -Os and branch profiling enabled.
         *
         * Work around it by just copying the first byte again...
         */
        version = policy.version;
        if (copy_from_user(&policy, arg, size))
                return -EFAULT;
        policy.version = version;

        if (!inode_owner_or_capable(&nop_mnt_idmap, inode))
                return -EACCES;

        ret = mnt_want_write_file(filp);
        if (ret)
                return ret;

        inode_lock(inode);

        ret = fscrypt_get_policy(inode, &existing_policy);
        if (ret == -ENODATA) {
                if (!S_ISDIR(inode->i_mode))
                        ret = -ENOTDIR;
                else if (IS_DEADDIR(inode))
                        ret = -ENOENT;
                else if (!inode->i_sb->s_cop->empty_dir(inode))
                        ret = -ENOTEMPTY;
                else
                        ret = set_encryption_policy(inode, &policy);
        } else if (ret == -EINVAL ||
                   (ret == 0 && !fscrypt_policies_equal(&policy,
                                                        &existing_policy))) {
                /* The file already uses a different encryption policy. */
                ret = -EEXIST;
        }

        inode_unlock(inode);

        mnt_drop_write_file(filp);
        return ret;
}
EXPORT_SYMBOL(fscrypt_ioctl_set_policy);

/* Original ioctl version; can only get the original policy version */
int fscrypt_ioctl_get_policy(struct file *filp, void __user *arg)
{
        union fscrypt_policy policy;
        int err;

        err = fscrypt_get_policy(file_inode(filp), &policy);
        if (err)
                return err;

        if (policy.version != FSCRYPT_POLICY_V1)
                return -EINVAL;

        if (copy_to_user(arg, &policy, sizeof(policy.v1)))
                return -EFAULT;
        return 0;
}
EXPORT_SYMBOL(fscrypt_ioctl_get_policy);

/* Extended ioctl version; can get policies of any version */
int fscrypt_ioctl_get_policy_ex(struct file *filp, void __user *uarg)
{
        struct fscrypt_get_policy_ex_arg arg;
        union fscrypt_policy *policy = (union fscrypt_policy *)&arg.policy;
        size_t policy_size;
        int err;

        /* arg is policy_size, then policy */
        BUILD_BUG_ON(offsetof(typeof(arg), policy_size) != 0);
        BUILD_BUG_ON(offsetofend(typeof(arg), policy_size) !=
                     offsetof(typeof(arg), policy));
        BUILD_BUG_ON(sizeof(arg.policy) != sizeof(*policy));

        err = fscrypt_get_policy(file_inode(filp), policy);
        if (err)
                return err;
        policy_size = fscrypt_policy_size(policy);

        if (copy_from_user(&arg, uarg, sizeof(arg.policy_size)))
                return -EFAULT;

        if (policy_size > arg.policy_size)
                return -EOVERFLOW;
        arg.policy_size = policy_size;

        if (copy_to_user(uarg, &arg, sizeof(arg.policy_size) + policy_size))
                return -EFAULT;
        return 0;
}
EXPORT_SYMBOL_GPL(fscrypt_ioctl_get_policy_ex);

/* FS_IOC_GET_ENCRYPTION_NONCE: retrieve file's encryption nonce for testing */
int fscrypt_ioctl_get_nonce(struct file *filp, void __user *arg)
{
        struct inode *inode = file_inode(filp);
        union fscrypt_context ctx;
        int ret;

        ret = inode->i_sb->s_cop->get_context(inode, &ctx, sizeof(ctx));
        if (ret < 0)
                return ret;
        if (!fscrypt_context_is_valid(&ctx, ret))
                return -EINVAL;
        if (copy_to_user(arg, fscrypt_context_nonce(&ctx),
                         FSCRYPT_FILE_NONCE_SIZE))
                return -EFAULT;
        return 0;
}
EXPORT_SYMBOL_GPL(fscrypt_ioctl_get_nonce);

/**
 * fscrypt_has_permitted_context() - is a file's encryption policy permitted
 *                                     within its directory?
 *
 * @parent: inode for parent directory
 * @child: inode for file being looked up, opened, or linked into @parent
 *
 * Filesystems must call this before permitting access to an inode in a
 * situation where the parent directory is encrypted (either before allowing
 * ->lookup() to succeed, or for a regular file before allowing it to be opened)
 * and before any operation that involves linking an inode into an encrypted
 * directory, including link, rename, and cross rename.  It enforces the
 * constraint that within a given encrypted directory tree, all files use the
 * same encryption policy.  The pre-access check is needed to detect potentially
 * malicious offline violations of this constraint, while the link and rename
 * checks are needed to prevent online violations of this constraint.
 *
 * Return: 1 if permitted, 0 if forbidden.
 */
int fscrypt_has_permitted_context(struct inode *parent, struct inode *child)
{
        union fscrypt_policy parent_policy, child_policy;
        int err, err1, err2;

        /* No restrictions on file types which are never encrypted */
        if (!S_ISREG(child->i_mode) && !S_ISDIR(child->i_mode) &&
            !S_ISLNK(child->i_mode))
                return 1;

        /* No restrictions if the parent directory is unencrypted */
        if (!IS_ENCRYPTED(parent))
                return 1;

        /* Encrypted directories must not contain unencrypted files */
        if (!IS_ENCRYPTED(child))
                return 0;

        /*
         * Both parent and child are encrypted, so verify they use the same
         * encryption policy.  Compare the cached policies if the keys are
         * available, otherwise retrieve and compare the fscrypt_contexts.
         *
         * Note that the fscrypt_context retrieval will be required frequently
         * when accessing an encrypted directory tree without the key.
         * Performance-wise this is not a big deal because we already don't
         * really optimize for file access without the key (to the extent that
         * such access is even possible), given that any attempted access
         * already causes a fscrypt_context retrieval and keyring search.
         *
         * In any case, if an unexpected error occurs, fall back to "forbidden".
         */

        err = fscrypt_get_encryption_info(parent, true);
        if (err)
                return 0;
        err = fscrypt_get_encryption_info(child, true);
        if (err)
                return 0;

        err1 = fscrypt_get_policy(parent, &parent_policy);
        err2 = fscrypt_get_policy(child, &child_policy);

        /*
         * Allow the case where the parent and child both have an unrecognized
         * encryption policy, so that files with an unrecognized encryption
         * policy can be deleted.
         */
        if (err1 == -EINVAL && err2 == -EINVAL)
                return 1;

        if (err1 || err2)
                return 0;

        return fscrypt_policies_equal(&parent_policy, &child_policy);
}
EXPORT_SYMBOL(fscrypt_has_permitted_context);

/*
 * Return the encryption policy that new files in the directory will inherit, or
 * NULL if none, or an ERR_PTR() on error.  If the directory is encrypted, also
 * ensure that its key is set up, so that the new filename can be encrypted.
 */
const union fscrypt_policy *fscrypt_policy_to_inherit(struct inode *dir)
{
        int err;

        if (IS_ENCRYPTED(dir)) {
                err = fscrypt_require_key(dir);
                if (err)
                        return ERR_PTR(err);
                return &dir->i_crypt_info->ci_policy;
        }

        return fscrypt_get_dummy_policy(dir->i_sb);
}

/**
 * fscrypt_context_for_new_inode() - create an encryption context for a new inode
 * @ctx: where context should be written
 * @inode: inode from which to fetch policy and nonce
 *
 * Given an in-core "prepared" (via fscrypt_prepare_new_inode) inode,
 * generate a new context and write it to ctx. ctx _must_ be at least
 * FSCRYPT_SET_CONTEXT_MAX_SIZE bytes.
 *
 * Return: size of the resulting context or a negative error code.
 */
int fscrypt_context_for_new_inode(void *ctx, struct inode *inode)
{
        struct fscrypt_inode_info *ci = inode->i_crypt_info;

        BUILD_BUG_ON(sizeof(union fscrypt_context) !=
                        FSCRYPT_SET_CONTEXT_MAX_SIZE);

        /* fscrypt_prepare_new_inode() should have set up the key already. */
        if (WARN_ON_ONCE(!ci))
                return -ENOKEY;

        return fscrypt_new_context(ctx, &ci->ci_policy, ci->ci_nonce);
}
EXPORT_SYMBOL_GPL(fscrypt_context_for_new_inode);

/**
 * fscrypt_set_context() - Set the fscrypt context of a new inode
 * @inode: a new inode
 * @fs_data: private data given by FS and passed to ->set_context()
 *
 * This should be called after fscrypt_prepare_new_inode(), generally during a
 * filesystem transaction.  Everything here must be %GFP_NOFS-safe.
 *
 * Return: 0 on success, -errno on failure
 */
int fscrypt_set_context(struct inode *inode, void *fs_data)
{
        struct fscrypt_inode_info *ci = inode->i_crypt_info;
        union fscrypt_context ctx;
        int ctxsize;

        ctxsize = fscrypt_context_for_new_inode(&ctx, inode);
        if (ctxsize < 0)
                return ctxsize;

        /*
         * This may be the first time the inode number is available, so do any
         * delayed key setup that requires the inode number.
         */
        if (ci->ci_policy.version == FSCRYPT_POLICY_V2 &&
            (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32))
                fscrypt_hash_inode_number(ci, ci->ci_master_key);

        return inode->i_sb->s_cop->set_context(inode, &ctx, ctxsize, fs_data);
}
EXPORT_SYMBOL_GPL(fscrypt_set_context);

/**
 * fscrypt_parse_test_dummy_encryption() - parse the test_dummy_encryption mount option
 * @param: the mount option
 * @dummy_policy: (input/output) the place to write the dummy policy that will
 *        result from parsing the option.  Zero-initialize this.  If a policy is
 *        already set here (due to test_dummy_encryption being given multiple
 *        times), then this function will verify that the policies are the same.
 *
 * Return: 0 on success; -EINVAL if the argument is invalid; -EEXIST if the
 *           argument conflicts with one already specified; or -ENOMEM.
 */
int fscrypt_parse_test_dummy_encryption(const struct fs_parameter *param,
                                struct fscrypt_dummy_policy *dummy_policy)
{
        const char *arg = "v2";
        union fscrypt_policy *policy;
        int err;

        if (param->type == fs_value_is_string && *param->string)
                arg = param->string;

        policy = kzalloc(sizeof(*policy), GFP_KERNEL);
        if (!policy)
                return -ENOMEM;

        if (!strcmp(arg, "v1")) {
                policy->version = FSCRYPT_POLICY_V1;
                policy->v1.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS;
                policy->v1.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS;
                memset(policy->v1.master_key_descriptor, 0x42,
                       FSCRYPT_KEY_DESCRIPTOR_SIZE);
        } else if (!strcmp(arg, "v2")) {
                policy->version = FSCRYPT_POLICY_V2;
                policy->v2.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS;
                policy->v2.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS;
                err = fscrypt_get_test_dummy_key_identifier(
                                policy->v2.master_key_identifier);
                if (err)
                        goto out;
        } else {
                err = -EINVAL;
                goto out;
        }

        if (dummy_policy->policy) {
                if (fscrypt_policies_equal(policy, dummy_policy->policy))
                        err = 0;
                else
                        err = -EEXIST;
                goto out;
        }
        dummy_policy->policy = policy;
        policy = NULL;
        err = 0;
out:
        kfree(policy);
        return err;
}
EXPORT_SYMBOL_GPL(fscrypt_parse_test_dummy_encryption);

/**
 * fscrypt_dummy_policies_equal() - check whether two dummy policies are equal
 * @p1: the first test dummy policy (may be unset)
 * @p2: the second test dummy policy (may be unset)
 *
 * Return: %true if the dummy policies are both set and equal, or both unset.
 */
bool fscrypt_dummy_policies_equal(const struct fscrypt_dummy_policy *p1,
                                  const struct fscrypt_dummy_policy *p2)
{
        if (!p1->policy && !p2->policy)
                return true;
        if (!p1->policy || !p2->policy)
                return false;
        return fscrypt_policies_equal(p1->policy, p2->policy);
}
EXPORT_SYMBOL_GPL(fscrypt_dummy_policies_equal);

/**
 * fscrypt_show_test_dummy_encryption() - show '-o test_dummy_encryption'
 * @seq: the seq_file to print the option to
 * @sep: the separator character to use
 * @sb: the filesystem whose options are being shown
 *
 * Show the test_dummy_encryption mount option, if it was specified.
 * This is mainly used for /proc/mounts.
 */
void fscrypt_show_test_dummy_encryption(struct seq_file *seq, char sep,
                                        struct super_block *sb)
{
        const union fscrypt_policy *policy = fscrypt_get_dummy_policy(sb);
        int vers;

        if (!policy)
                return;

        vers = policy->version;
        if (vers == FSCRYPT_POLICY_V1) /* Handle numbering quirk */
                vers = 1;

        seq_printf(seq, "%ctest_dummy_encryption=v%d", sep, vers);
}
EXPORT_SYMBOL_GPL(fscrypt_show_test_dummy_encryption);











































































































































































































































 1860 












 1889 










 1891 
    1 



 1889 
   12 
























 1890 
 1878 
   11 




 1886 







 1846 



 1838 
 1846 
   12 
 1845 
    1 
 1846 





 1856 








 1859 




 1855 


 1857 





 1856 

 1857 



 1855 

 1859 



 1857 








 1118 












 1857 
  290 









 1855 
    4 






 1854 



  978 



   51 

  980 



 1602 





  238 


 1604 
  449 
 1601 

 1855 






  974 










   27 






  978 
   56 





  976 
  976 

  976 


  714 
  715 










  713 










  977 



  874 
  115 
  110 


  114 



  978 






   56 


  977 
  738 
  975 


  948 

  979 







































   63 


   63 




   53 


   53 













   53 






   53 
   63 





































   32 


   36 



   36 







   23 

    9 










 1906 
 1899 


 1869 
 1585 




 1868 
   39 


 1866 
   39 


 1864 
 1856 


 1862 
   13 


 1861 
    4 













 1855 





 1852 

 1855 
 1833 


 1857 
  366 


 1857 
  245 


   24 
   41 


 1858 










 1855 


 1853 
 1856 





















































































 1852 







 1860 
 1859 
 1860 










    4 


 1893 
 1892 










 1888 
 1882 



 1893 

 1886 
 1893 







 1892 




 1891 
 1896 

 1887 






 1887 


 1893 

 1857 





 1858 


 1857 
 1858 


 1856 














 1933 
 1922 
 1888 


 1893 

 1934 


 1935 




 1934 








 1936 






 1936 




























 1898 













































































































































































    2 
    2 
    2 
    2 











































    2 
    2 
    2 














    2 






































   49 




   49 
   49 




   49 
   48 
   49 
    1 























   56 



   56 



   56 

   56 
   56 













   49 

   31 

















   56 



   56 






   56 








   52 

   52 


    1 
    1 









   52 






































   49 










   49 
   49 

   49 
   49 








   49 
   49 







    1 






    1 




























   49 

   49 






   49 




























   90 



   56 



   56 


   56 
   56 
   56 




   55 




   56 
   90 











































































































   82 














 1898 























 1935 

 1855 




























 1945 





 1938 



















 1954 



























   12 





















 1942 


 1948 


























 1953 





 1956 
























   14 







   14 














 1964 












   86 






   51 
   51 
   51 








   34 

   33 








    2 
    2 





   50 
    1 








    7 





   49 
   85 




   69 









   49 


   49 








   69 






   90 











   36 







   35 



















    7 

    7 






 1842 
 1842 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
// SPDX-License-Identifier: GPL-2.0
/*
 * drivers/usb/core/driver.c - most of the driver model stuff for usb
 *
 * (C) Copyright 2005 Greg Kroah-Hartman <gregkh@suse.de>
 *
 * based on drivers/usb/usb.c which had the following copyrights:
 *        (C) Copyright Linus Torvalds 1999
 *        (C) Copyright Johannes Erdfelt 1999-2001
 *        (C) Copyright Andreas Gal 1999
 *        (C) Copyright Gregory P. Smith 1999
 *        (C) Copyright Deti Fliegl 1999 (new USB architecture)
 *        (C) Copyright Randy Dunlap 2000
 *        (C) Copyright David Brownell 2000-2004
 *        (C) Copyright Yggdrasil Computing, Inc. 2000
 *                (usb_device_id matching changes by Adam J. Richter)
 *        (C) Copyright Greg Kroah-Hartman 2002-2003
 *
 * Released under the GPLv2 only.
 *
 * NOTE! This is not actually a driver at all, rather this is
 * just a collection of helper routines that implement the
 * matching, probing, releasing, suspending and resuming for
 * real drivers.
 *
 */

#include <linux/device.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/usb.h>
#include <linux/usb/quirks.h>
#include <linux/usb/hcd.h>

#include "usb.h"


/*
 * Adds a new dynamic USBdevice ID to this driver,
 * and cause the driver to probe for all devices again.
 */
ssize_t usb_store_new_id(struct usb_dynids *dynids,
                         const struct usb_device_id *id_table,
                         struct device_driver *driver,
                         const char *buf, size_t count)
{
        struct usb_dynid *dynid;
        u32 idVendor = 0;
        u32 idProduct = 0;
        unsigned int bInterfaceClass = 0;
        u32 refVendor, refProduct;
        int fields = 0;
        int retval = 0;

        fields = sscanf(buf, "%x %x %x %x %x", &idVendor, &idProduct,
                        &bInterfaceClass, &refVendor, &refProduct);
        if (fields < 2)
                return -EINVAL;

        dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
        if (!dynid)
                return -ENOMEM;

        INIT_LIST_HEAD(&dynid->node);
        dynid->id.idVendor = idVendor;
        dynid->id.idProduct = idProduct;
        dynid->id.match_flags = USB_DEVICE_ID_MATCH_DEVICE;
        if (fields > 2 && bInterfaceClass) {
                if (bInterfaceClass > 255) {
                        retval = -EINVAL;
                        goto fail;
                }

                dynid->id.bInterfaceClass = (u8)bInterfaceClass;
                dynid->id.match_flags |= USB_DEVICE_ID_MATCH_INT_CLASS;
        }

        if (fields > 4) {
                const struct usb_device_id *id = id_table;

                if (!id) {
                        retval = -ENODEV;
                        goto fail;
                }

                for (; id->match_flags; id++)
                        if (id->idVendor == refVendor && id->idProduct == refProduct)
                                break;

                if (id->match_flags) {
                        dynid->id.driver_info = id->driver_info;
                } else {
                        retval = -ENODEV;
                        goto fail;
                }
        }

        mutex_lock(&usb_dynids_lock);
        list_add_tail(&dynid->node, &dynids->list);
        mutex_unlock(&usb_dynids_lock);

        retval = driver_attach(driver);

        if (retval)
                return retval;
        return count;

fail:
        kfree(dynid);
        return retval;
}
EXPORT_SYMBOL_GPL(usb_store_new_id);

ssize_t usb_show_dynids(struct usb_dynids *dynids, char *buf)
{
        struct usb_dynid *dynid;
        size_t count = 0;

        guard(mutex)(&usb_dynids_lock);
        list_for_each_entry(dynid, &dynids->list, node)
                if (dynid->id.bInterfaceClass != 0)
                        count += scnprintf(&buf[count], PAGE_SIZE - count, "%04x %04x %02x\n",
                                           dynid->id.idVendor, dynid->id.idProduct,
                                           dynid->id.bInterfaceClass);
                else
                        count += scnprintf(&buf[count], PAGE_SIZE - count, "%04x %04x\n",
                                           dynid->id.idVendor, dynid->id.idProduct);
        return count;
}
EXPORT_SYMBOL_GPL(usb_show_dynids);

static ssize_t new_id_show(struct device_driver *driver, char *buf)
{
        struct usb_driver *usb_drv = to_usb_driver(driver);

        return usb_show_dynids(&usb_drv->dynids, buf);
}

static ssize_t new_id_store(struct device_driver *driver,
                            const char *buf, size_t count)
{
        struct usb_driver *usb_drv = to_usb_driver(driver);

        return usb_store_new_id(&usb_drv->dynids, usb_drv->id_table, driver, buf, count);
}
static DRIVER_ATTR_RW(new_id);

/*
 * Remove a USB device ID from this driver
 */
static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
                               size_t count)
{
        struct usb_dynid *dynid, *n;
        struct usb_driver *usb_driver = to_usb_driver(driver);
        u32 idVendor;
        u32 idProduct;
        int fields;

        fields = sscanf(buf, "%x %x", &idVendor, &idProduct);
        if (fields < 2)
                return -EINVAL;

        guard(mutex)(&usb_dynids_lock);
        list_for_each_entry_safe(dynid, n, &usb_driver->dynids.list, node) {
                struct usb_device_id *id = &dynid->id;

                if ((id->idVendor == idVendor) &&
                    (id->idProduct == idProduct)) {
                        list_del(&dynid->node);
                        kfree(dynid);
                        break;
                }
        }
        return count;
}

static ssize_t remove_id_show(struct device_driver *driver, char *buf)
{
        return new_id_show(driver, buf);
}
static DRIVER_ATTR_RW(remove_id);

static int usb_create_newid_files(struct usb_driver *usb_drv)
{
        int error = 0;

        if (usb_drv->no_dynamic_id)
                goto exit;

        if (usb_drv->probe != NULL) {
                error = driver_create_file(&usb_drv->driver,
                                           &driver_attr_new_id);
                if (error == 0) {
                        error = driver_create_file(&usb_drv->driver,
                                        &driver_attr_remove_id);
                        if (error)
                                driver_remove_file(&usb_drv->driver,
                                                &driver_attr_new_id);
                }
        }
exit:
        return error;
}

static void usb_remove_newid_files(struct usb_driver *usb_drv)
{
        if (usb_drv->no_dynamic_id)
                return;

        if (usb_drv->probe != NULL) {
                driver_remove_file(&usb_drv->driver,
                                &driver_attr_remove_id);
                driver_remove_file(&usb_drv->driver,
                                   &driver_attr_new_id);
        }
}

static void usb_free_dynids(struct usb_driver *usb_drv)
{
        struct usb_dynid *dynid, *n;

        guard(mutex)(&usb_dynids_lock);
        list_for_each_entry_safe(dynid, n, &usb_drv->dynids.list, node) {
                list_del(&dynid->node);
                kfree(dynid);
        }
}

static const struct usb_device_id *usb_match_dynamic_id(struct usb_interface *intf,
                                                        const struct usb_driver *drv)
{
        struct usb_dynid *dynid;

        guard(mutex)(&usb_dynids_lock);
        list_for_each_entry(dynid, &drv->dynids.list, node) {
                if (usb_match_one_id(intf, &dynid->id)) {
                        return &dynid->id;
                }
        }
        return NULL;
}


/* called from driver core with dev locked */
static int usb_probe_device(struct device *dev)
{
        struct usb_device_driver *udriver = to_usb_device_driver(dev->driver);
        struct usb_device *udev = to_usb_device(dev);
        int error = 0;

        dev_dbg(dev, "%s\n", __func__);

        /* TODO: Add real matching code */

        /* The device should always appear to be in use
         * unless the driver supports autosuspend.
         */
        if (!udriver->supports_autosuspend)
                error = usb_autoresume_device(udev);
        if (error)
                return error;

        if (udriver->generic_subclass)
                error = usb_generic_driver_probe(udev);
        if (error)
                return error;

        /* Probe the USB device with the driver in hand, but only
         * defer to a generic driver in case the current USB
         * device driver has an id_table or a match function; i.e.,
         * when the device driver was explicitly matched against
         * a device.
         *
         * If the device driver does not have either of these,
         * then we assume that it can bind to any device and is
         * not truly a more specialized/non-generic driver, so a
         * return value of -ENODEV should not force the device
         * to be handled by the generic USB driver, as there
         * can still be another, more specialized, device driver.
         *
         * This accommodates the usbip driver.
         *
         * TODO: What if, in the future, there are multiple
         * specialized USB device drivers for a particular device?
         * In such cases, there is a need to try all matching
         * specialised device drivers prior to setting the
         * use_generic_driver bit.
         */
        if (udriver->probe)
                error = udriver->probe(udev);
        else if (!udriver->generic_subclass)
                error = -EINVAL;
        if (error == -ENODEV && udriver != &usb_generic_driver &&
            (udriver->id_table || udriver->match)) {
                udev->use_generic_driver = 1;
                return -EPROBE_DEFER;
        }
        return error;
}

/* called from driver core with dev locked */
static int usb_unbind_device(struct device *dev)
{
        struct usb_device *udev = to_usb_device(dev);
        struct usb_device_driver *udriver = to_usb_device_driver(dev->driver);

        if (udriver->disconnect)
                udriver->disconnect(udev);
        if (udriver->generic_subclass)
                usb_generic_driver_disconnect(udev);
        if (!udriver->supports_autosuspend)
                usb_autosuspend_device(udev);
        return 0;
}

/* called from driver core with dev locked */
static int usb_probe_interface(struct device *dev)
{
        struct usb_driver *driver = to_usb_driver(dev->driver);
        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_device *udev = interface_to_usbdev(intf);
        const struct usb_device_id *id;
        int error = -ENODEV;
        int lpm_disable_error = -ENODEV;

        dev_dbg(dev, "%s\n", __func__);

        intf->needs_binding = 0;

        if (usb_device_is_owned(udev))
                return error;

        if (udev->authorized == 0) {
                dev_err(&intf->dev, "Device is not authorized for usage\n");
                return error;
        } else if (intf->authorized == 0) {
                dev_err(&intf->dev, "Interface %d is not authorized for usage\n",
                                intf->altsetting->desc.bInterfaceNumber);
                return error;
        }

        id = usb_match_dynamic_id(intf, driver);
        if (!id)
                id = usb_match_id(intf, driver->id_table);
        if (!id)
                return error;

        dev_dbg(dev, "%s - got id\n", __func__);

        error = usb_autoresume_device(udev);
        if (error)
                return error;

        intf->condition = USB_INTERFACE_BINDING;

        /* Probed interfaces are initially active.  They are
         * runtime-PM-enabled only if the driver has autosuspend support.
         * They are sensitive to their children's power states.
         */
        pm_runtime_set_active(dev);
        pm_suspend_ignore_children(dev, false);
        if (driver->supports_autosuspend)
                pm_runtime_enable(dev);

        /* If the new driver doesn't allow hub-initiated LPM, and we can't
         * disable hub-initiated LPM, then fail the probe.
         *
         * Otherwise, leaving LPM enabled should be harmless, because the
         * endpoint intervals should remain the same, and the U1/U2 timeouts
         * should remain the same.
         *
         * If we need to install alt setting 0 before probe, or another alt
         * setting during probe, that should also be fine.  usb_set_interface()
         * will attempt to disable LPM, and fail if it can't disable it.
         */
        if (driver->disable_hub_initiated_lpm) {
                lpm_disable_error = usb_unlocked_disable_lpm(udev);
                if (lpm_disable_error) {
                        dev_err(&intf->dev, "%s Failed to disable LPM for driver %s\n",
                                __func__, driver->name);
                        error = lpm_disable_error;
                        goto err;
                }
        }

        /* Carry out a deferred switch to altsetting 0 */
        if (intf->needs_altsetting0) {
                error = usb_set_interface(udev, intf->altsetting[0].
                                desc.bInterfaceNumber, 0);
                if (error < 0)
                        goto err;
                intf->needs_altsetting0 = 0;
        }

        error = driver->probe(intf, id);
        if (error)
                goto err;

        intf->condition = USB_INTERFACE_BOUND;

        /* If the LPM disable succeeded, balance the ref counts. */
        if (!lpm_disable_error)
                usb_unlocked_enable_lpm(udev);

        usb_autosuspend_device(udev);
        return error;

 err:
        usb_set_intfdata(intf, NULL);
        intf->needs_remote_wakeup = 0;
        intf->condition = USB_INTERFACE_UNBOUND;

        /* If the LPM disable succeeded, balance the ref counts. */
        if (!lpm_disable_error)
                usb_unlocked_enable_lpm(udev);

        /* Unbound interfaces are always runtime-PM-disabled and -suspended */
        if (driver->supports_autosuspend)
                pm_runtime_disable(dev);
        pm_runtime_set_suspended(dev);

        usb_autosuspend_device(udev);
        return error;
}

/* called from driver core with dev locked */
static int usb_unbind_interface(struct device *dev)
{
        struct usb_driver *driver = to_usb_driver(dev->driver);
        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_host_endpoint *ep, **eps = NULL;
        struct usb_device *udev;
        int i, j, error, r;
        int lpm_disable_error = -ENODEV;

        intf->condition = USB_INTERFACE_UNBINDING;

        /* Autoresume for set_interface call below */
        udev = interface_to_usbdev(intf);
        error = usb_autoresume_device(udev);

        /* If hub-initiated LPM policy may change, attempt to disable LPM until
         * the driver is unbound.  If LPM isn't disabled, that's fine because it
         * wouldn't be enabled unless all the bound interfaces supported
         * hub-initiated LPM.
         */
        if (driver->disable_hub_initiated_lpm)
                lpm_disable_error = usb_unlocked_disable_lpm(udev);

        /*
         * Terminate all URBs for this interface unless the driver
         * supports "soft" unbinding and the device is still present.
         */
        if (!driver->soft_unbind || udev->state == USB_STATE_NOTATTACHED)
                usb_disable_interface(udev, intf, false);

        driver->disconnect(intf);

        /* Free streams */
        for (i = 0, j = 0; i < intf->cur_altsetting->desc.bNumEndpoints; i++) {
                ep = &intf->cur_altsetting->endpoint[i];
                if (ep->streams == 0)
                        continue;
                if (j == 0) {
                        eps = kmalloc_array(USB_MAXENDPOINTS, sizeof(void *),
                                      GFP_KERNEL);
                        if (!eps)
                                break;
                }
                eps[j++] = ep;
        }
        if (j) {
                usb_free_streams(intf, eps, j, GFP_KERNEL);
                kfree(eps);
        }

        /* Reset other interface state.
         * We cannot do a Set-Interface if the device is suspended or
         * if it is prepared for a system sleep (since installing a new
         * altsetting means creating new endpoint device entries).
         * When either of these happens, defer the Set-Interface.
         */
        if (intf->cur_altsetting->desc.bAlternateSetting == 0) {
                /* Already in altsetting 0 so skip Set-Interface.
                 * Just re-enable it without affecting the endpoint toggles.
                 */
                usb_enable_interface(udev, intf, false);
        } else if (!error && !intf->dev.power.is_prepared) {
                r = usb_set_interface(udev, intf->altsetting[0].
                                desc.bInterfaceNumber, 0);
                if (r < 0)
                        intf->needs_altsetting0 = 1;
        } else {
                intf->needs_altsetting0 = 1;
        }
        usb_set_intfdata(intf, NULL);

        intf->condition = USB_INTERFACE_UNBOUND;
        intf->needs_remote_wakeup = 0;

        /* Attempt to re-enable USB3 LPM, if the disable succeeded. */
        if (!lpm_disable_error)
                usb_unlocked_enable_lpm(udev);

        /* Unbound interfaces are always runtime-PM-disabled and -suspended */
        if (driver->supports_autosuspend)
                pm_runtime_disable(dev);
        pm_runtime_set_suspended(dev);

        if (!error)
                usb_autosuspend_device(udev);

        return 0;
}

static void usb_shutdown_interface(struct device *dev)
{
        struct usb_interface *intf = to_usb_interface(dev);
        struct usb_driver *driver;

        if (!dev->driver)
                return;

        driver = to_usb_driver(dev->driver);
        if (driver->shutdown)
                driver->shutdown(intf);
}

/**
 * usb_driver_claim_interface - bind a driver to an interface
 * @driver: the driver to be bound
 * @iface: the interface to which it will be bound; must be in the
 *        usb device's active configuration
 * @data: driver data associated with that interface
 *
 * This is used by usb device drivers that need to claim more than one
 * interface on a device when probing (audio and acm are current examples).
 * No device driver should directly modify internal usb_interface or
 * usb_device structure members.
 *
 * Callers must own the device lock, so driver probe() entries don't need
 * extra locking, but other call contexts may need to explicitly claim that
 * lock.
 *
 * Return: 0 on success.
 */
int usb_driver_claim_interface(struct usb_driver *driver,
                                struct usb_interface *iface, void *data)
{
        struct device *dev;
        int retval = 0;

        if (!iface)
                return -ENODEV;

        dev = &iface->dev;
        if (dev->driver)
                return -EBUSY;

        /* reject claim if interface is not authorized */
        if (!iface->authorized)
                return -ENODEV;

        dev->driver = &driver->driver;
        usb_set_intfdata(iface, data);
        iface->needs_binding = 0;

        iface->condition = USB_INTERFACE_BOUND;

        /* Claimed interfaces are initially inactive (suspended) and
         * runtime-PM-enabled, but only if the driver has autosuspend
         * support.  Otherwise they are marked active, to prevent the
         * device from being autosuspended, but left disabled.  In either
         * case they are sensitive to their children's power states.
         */
        pm_suspend_ignore_children(dev, false);
        if (driver->supports_autosuspend)
                pm_runtime_enable(dev);
        else
                pm_runtime_set_active(dev);

        /* if interface was already added, bind now; else let
         * the future device_add() bind it, bypassing probe()
         */
        if (device_is_registered(dev))
                retval = device_bind_driver(dev);

        if (retval) {
                dev->driver = NULL;
                usb_set_intfdata(iface, NULL);
                iface->needs_remote_wakeup = 0;
                iface->condition = USB_INTERFACE_UNBOUND;

                /*
                 * Unbound interfaces are always runtime-PM-disabled
                 * and runtime-PM-suspended
                 */
                if (driver->supports_autosuspend)
                        pm_runtime_disable(dev);
                pm_runtime_set_suspended(dev);
        }

        return retval;
}
EXPORT_SYMBOL_GPL(usb_driver_claim_interface);

/**
 * usb_driver_release_interface - unbind a driver from an interface
 * @driver: the driver to be unbound
 * @iface: the interface from which it will be unbound
 *
 * This can be used by drivers to release an interface without waiting
 * for their disconnect() methods to be called.  In typical cases this
 * also causes the driver disconnect() method to be called.
 *
 * This call is synchronous, and may not be used in an interrupt context.
 * Callers must own the device lock, so driver disconnect() entries don't
 * need extra locking, but other call contexts may need to explicitly claim
 * that lock.
 */
void usb_driver_release_interface(struct usb_driver *driver,
                                        struct usb_interface *iface)
{
        struct device *dev = &iface->dev;

        /* this should never happen, don't release something that's not ours */
        if (!dev->driver || dev->driver != &driver->driver)
                return;

        /* don't release from within disconnect() */
        if (iface->condition != USB_INTERFACE_BOUND)
                return;
        iface->condition = USB_INTERFACE_UNBINDING;

        /* Release via the driver core only if the interface
         * has already been registered
         */
        if (device_is_registered(dev)) {
                device_release_driver(dev);
        } else {
                device_lock(dev);
                usb_unbind_interface(dev);
                dev->driver = NULL;
                device_unlock(dev);
        }
}
EXPORT_SYMBOL_GPL(usb_driver_release_interface);

/* returns 0 if no match, 1 if match */
int usb_match_device(struct usb_device *dev, const struct usb_device_id *id)
{
        if ((id->match_flags & USB_DEVICE_ID_MATCH_VENDOR) &&
            id->idVendor != le16_to_cpu(dev->descriptor.idVendor))
                return 0;

        if ((id->match_flags & USB_DEVICE_ID_MATCH_PRODUCT) &&
            id->idProduct != le16_to_cpu(dev->descriptor.idProduct))
                return 0;

        /* No need to test id->bcdDevice_lo != 0, since 0 is never
           greater than any unsigned number. */
        if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_LO) &&
            (id->bcdDevice_lo > le16_to_cpu(dev->descriptor.bcdDevice)))
                return 0;

        if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_HI) &&
            (id->bcdDevice_hi < le16_to_cpu(dev->descriptor.bcdDevice)))
                return 0;

        if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_CLASS) &&
            (id->bDeviceClass != dev->descriptor.bDeviceClass))
                return 0;

        if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_SUBCLASS) &&
            (id->bDeviceSubClass != dev->descriptor.bDeviceSubClass))
                return 0;

        if ((id->match_flags & USB_DEVICE_ID_MATCH_DEV_PROTOCOL) &&
            (id->bDeviceProtocol != dev->descriptor.bDeviceProtocol))
                return 0;

        return 1;
}

/* returns 0 if no match, 1 if match */
int usb_match_one_id_intf(struct usb_device *dev,
                          struct usb_host_interface *intf,
                          const struct usb_device_id *id)
{
        /* The interface class, subclass, protocol and number should never be
         * checked for a match if the device class is Vendor Specific,
         * unless the match record specifies the Vendor ID. */
        if (dev->descriptor.bDeviceClass == USB_CLASS_VENDOR_SPEC &&
                        !(id->match_flags & USB_DEVICE_ID_MATCH_VENDOR) &&
                        (id->match_flags & (USB_DEVICE_ID_MATCH_INT_CLASS |
                                USB_DEVICE_ID_MATCH_INT_SUBCLASS |
                                USB_DEVICE_ID_MATCH_INT_PROTOCOL |
                                USB_DEVICE_ID_MATCH_INT_NUMBER)))
                return 0;

        if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_CLASS) &&
            (id->bInterfaceClass != intf->desc.bInterfaceClass))
                return 0;

        if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_SUBCLASS) &&
            (id->bInterfaceSubClass != intf->desc.bInterfaceSubClass))
                return 0;

        if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_PROTOCOL) &&
            (id->bInterfaceProtocol != intf->desc.bInterfaceProtocol))
                return 0;

        if ((id->match_flags & USB_DEVICE_ID_MATCH_INT_NUMBER) &&
            (id->bInterfaceNumber != intf->desc.bInterfaceNumber))
                return 0;

        return 1;
}

/* returns 0 if no match, 1 if match */
int usb_match_one_id(struct usb_interface *interface,
                     const struct usb_device_id *id)
{
        struct usb_host_interface *intf;
        struct usb_device *dev;

        /* proc_connectinfo in devio.c may call us with id == NULL. */
        if (id == NULL)
                return 0;

        intf = interface->cur_altsetting;
        dev = interface_to_usbdev(interface);

        if (!usb_match_device(dev, id))
                return 0;

        return usb_match_one_id_intf(dev, intf, id);
}
EXPORT_SYMBOL_GPL(usb_match_one_id);

/**
 * usb_match_id - find first usb_device_id matching device or interface
 * @interface: the interface of interest
 * @id: array of usb_device_id structures, terminated by zero entry
 *
 * usb_match_id searches an array of usb_device_id's and returns
 * the first one matching the device or interface, or null.
 * This is used when binding (or rebinding) a driver to an interface.
 * Most USB device drivers will use this indirectly, through the usb core,
 * but some layered driver frameworks use it directly.
 * These device tables are exported with MODULE_DEVICE_TABLE, through
 * modutils, to support the driver loading functionality of USB hotplugging.
 *
 * Return: The first matching usb_device_id, or %NULL.
 *
 * What Matches:
 *
 * The "match_flags" element in a usb_device_id controls which
 * members are used.  If the corresponding bit is set, the
 * value in the device_id must match its corresponding member
 * in the device or interface descriptor, or else the device_id
 * does not match.
 *
 * "driver_info" is normally used only by device drivers,
 * but you can create a wildcard "matches anything" usb_device_id
 * as a driver's "modules.usbmap" entry if you provide an id with
 * only a nonzero "driver_info" field.  If you do this, the USB device
 * driver's probe() routine should use additional intelligence to
 * decide whether to bind to the specified interface.
 *
 * What Makes Good usb_device_id Tables:
 *
 * The match algorithm is very simple, so that intelligence in
 * driver selection must come from smart driver id records.
 * Unless you have good reasons to use another selection policy,
 * provide match elements only in related groups, and order match
 * specifiers from specific to general.  Use the macros provided
 * for that purpose if you can.
 *
 * The most specific match specifiers use device descriptor
 * data.  These are commonly used with product-specific matches;
 * the USB_DEVICE macro lets you provide vendor and product IDs,
 * and you can also match against ranges of product revisions.
 * These are widely used for devices with application or vendor
 * specific bDeviceClass values.
 *
 * Matches based on device class/subclass/protocol specifications
 * are slightly more general; use the USB_DEVICE_INFO macro, or
 * its siblings.  These are used with single-function devices
 * where bDeviceClass doesn't specify that each interface has
 * its own class.
 *
 * Matches based on interface class/subclass/protocol are the
 * most general; they let drivers bind to any interface on a
 * multiple-function device.  Use the USB_INTERFACE_INFO
 * macro, or its siblings, to match class-per-interface style
 * devices (as recorded in bInterfaceClass).
 *
 * Note that an entry created by USB_INTERFACE_INFO won't match
 * any interface if the device class is set to Vendor-Specific.
 * This is deliberate; according to the USB spec the meanings of
 * the interface class/subclass/protocol for these devices are also
 * vendor-specific, and hence matching against a standard product
 * class wouldn't work anyway.  If you really want to use an
 * interface-based match for such a device, create a match record
 * that also specifies the vendor ID.  (Unforunately there isn't a
 * standard macro for creating records like this.)
 *
 * Within those groups, remember that not all combinations are
 * meaningful.  For example, don't give a product version range
 * without vendor and product IDs; or specify a protocol without
 * its associated class and subclass.
 */
const struct usb_device_id *usb_match_id(struct usb_interface *interface,
                                         const struct usb_device_id *id)
{
        /* proc_connectinfo in devio.c may call us with id == NULL. */
        if (id == NULL)
                return NULL;

        /* It is important to check that id->driver_info is nonzero,
           since an entry that is all zeroes except for a nonzero
           id->driver_info is the way to create an entry that
           indicates that the driver want to examine every
           device and interface. */
        for (; id->idVendor || id->idProduct || id->bDeviceClass ||
               id->bInterfaceClass || id->driver_info; id++) {
                if (usb_match_one_id(interface, id))
                        return id;
        }

        return NULL;
}
EXPORT_SYMBOL_GPL(usb_match_id);

const struct usb_device_id *usb_device_match_id(struct usb_device *udev,
                                const struct usb_device_id *id)
{
        if (!id)
                return NULL;

        for (; id->idVendor || id->idProduct ; id++) {
                if (usb_match_device(udev, id))
                        return id;
        }

        return NULL;
}
EXPORT_SYMBOL_GPL(usb_device_match_id);

bool usb_driver_applicable(struct usb_device *udev,
                           const struct usb_device_driver *udrv)
{
        if (udrv->id_table && udrv->match)
                return usb_device_match_id(udev, udrv->id_table) != NULL &&
                       udrv->match(udev);

        if (udrv->id_table)
                return usb_device_match_id(udev, udrv->id_table) != NULL;

        if (udrv->match)
                return udrv->match(udev);

        return false;
}

static int usb_device_match(struct device *dev, const struct device_driver *drv)
{
        /* devices and interfaces are handled separately */
        if (is_usb_device(dev)) {
                struct usb_device *udev;
                const struct usb_device_driver *udrv;

                /* interface drivers never match devices */
                if (!is_usb_device_driver(drv))
                        return 0;

                udev = to_usb_device(dev);
                udrv = to_usb_device_driver(drv);

                /* If the device driver under consideration does not have a
                 * id_table or a match function, then let the driver's probe
                 * function decide.
                 */
                if (!udrv->id_table && !udrv->match)
                        return 1;

                return usb_driver_applicable(udev, udrv);

        } else if (is_usb_interface(dev)) {
                struct usb_interface *intf;
                const struct usb_driver *usb_drv;
                const struct usb_device_id *id;

                /* device drivers never match interfaces */
                if (is_usb_device_driver(drv))
                        return 0;

                intf = to_usb_interface(dev);
                usb_drv = to_usb_driver(drv);

                id = usb_match_id(intf, usb_drv->id_table);
                if (id)
                        return 1;

                id = usb_match_dynamic_id(intf, usb_drv);
                if (id)
                        return 1;
        }

        return 0;
}

static int usb_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
        const struct usb_device *usb_dev;

        if (is_usb_device(dev)) {
                usb_dev = to_usb_device(dev);
        } else if (is_usb_interface(dev)) {
                const struct usb_interface *intf = to_usb_interface(dev);

                usb_dev = interface_to_usbdev(intf);
        } else {
                return 0;
        }

        if (usb_dev->devnum < 0) {
                /* driver is often null here; dev_dbg() would oops */
                pr_debug("usb %s: already deleted?\n", dev_name(dev));
                return -ENODEV;
        }
        if (!usb_dev->bus) {
                pr_debug("usb %s: bus removed?\n", dev_name(dev));
                return -ENODEV;
        }

        /* per-device configurations are common */
        if (add_uevent_var(env, "PRODUCT=%x/%x/%x",
                           le16_to_cpu(usb_dev->descriptor.idVendor),
                           le16_to_cpu(usb_dev->descriptor.idProduct),
                           le16_to_cpu(usb_dev->descriptor.bcdDevice)))
                return -ENOMEM;

        /* class-based driver binding models */
        if (add_uevent_var(env, "TYPE=%d/%d/%d",
                           usb_dev->descriptor.bDeviceClass,
                           usb_dev->descriptor.bDeviceSubClass,
                           usb_dev->descriptor.bDeviceProtocol))
                return -ENOMEM;

        return 0;
}

static int __usb_bus_reprobe_drivers(struct device *dev, void *data)
{
        struct usb_device_driver *new_udriver = data;
        struct usb_device *udev;
        int ret;

        /* Don't reprobe if current driver isn't usb_generic_driver */
        if (dev->driver != &usb_generic_driver.driver)
                return 0;

        udev = to_usb_device(dev);
        if (!usb_driver_applicable(udev, new_udriver))
                return 0;

        ret = device_reprobe(dev);
        if (ret && ret != -EPROBE_DEFER)
                dev_err(dev, "Failed to reprobe device (error %d)\n", ret);

        return 0;
}

bool is_usb_device_driver(const struct device_driver *drv)
{
        return drv->probe == usb_probe_device;
}

/**
 * usb_register_device_driver - register a USB device (not interface) driver
 * @new_udriver: USB operations for the device driver
 * @owner: module owner of this driver.
 *
 * Registers a USB device driver with the USB core.  The list of
 * unattached devices will be rescanned whenever a new driver is
 * added, allowing the new driver to attach to any recognized devices.
 *
 * Return: A negative error code on failure and 0 on success.
 */
int usb_register_device_driver(struct usb_device_driver *new_udriver,
                struct module *owner)
{
        int retval = 0;

        if (usb_disabled())
                return -ENODEV;

        new_udriver->driver.name = new_udriver->name;
        new_udriver->driver.bus = &usb_bus_type;
        new_udriver->driver.probe = usb_probe_device;
        new_udriver->driver.remove = usb_unbind_device;
        new_udriver->driver.owner = owner;
        new_udriver->driver.dev_groups = new_udriver->dev_groups;

        retval = driver_register(&new_udriver->driver);

        if (!retval) {
                pr_info("%s: registered new device driver %s\n",
                        usbcore_name, new_udriver->name);
                /*
                 * Check whether any device could be better served with
                 * this new driver
                 */
                bus_for_each_dev(&usb_bus_type, NULL, new_udriver,
                                 __usb_bus_reprobe_drivers);
        } else {
                pr_err("%s: error %d registering device driver %s\n",
                        usbcore_name, retval, new_udriver->name);
        }

        return retval;
}
EXPORT_SYMBOL_GPL(usb_register_device_driver);

/**
 * usb_deregister_device_driver - unregister a USB device (not interface) driver
 * @udriver: USB operations of the device driver to unregister
 * Context: must be able to sleep
 *
 * Unlinks the specified driver from the internal USB driver list.
 */
void usb_deregister_device_driver(struct usb_device_driver *udriver)
{
        pr_info("%s: deregistering device driver %s\n",
                        usbcore_name, udriver->name);

        driver_unregister(&udriver->driver);
}
EXPORT_SYMBOL_GPL(usb_deregister_device_driver);

/**
 * usb_register_driver - register a USB interface driver
 * @new_driver: USB operations for the interface driver
 * @owner: module owner of this driver.
 * @mod_name: module name string
 *
 * Registers a USB interface driver with the USB core.  The list of
 * unattached interfaces will be rescanned whenever a new driver is
 * added, allowing the new driver to attach to any recognized interfaces.
 *
 * Return: A negative error code on failure and 0 on success.
 *
 * NOTE: if you want your driver to use the USB major number, you must call
 * usb_register_dev() to enable that functionality.  This function no longer
 * takes care of that.
 */
int usb_register_driver(struct usb_driver *new_driver, struct module *owner,
                        const char *mod_name)
{
        int retval = 0;

        if (usb_disabled())
                return -ENODEV;

        new_driver->driver.name = new_driver->name;
        new_driver->driver.bus = &usb_bus_type;
        new_driver->driver.probe = usb_probe_interface;
        new_driver->driver.remove = usb_unbind_interface;
        new_driver->driver.shutdown = usb_shutdown_interface;
        new_driver->driver.owner = owner;
        new_driver->driver.mod_name = mod_name;
        new_driver->driver.dev_groups = new_driver->dev_groups;
        INIT_LIST_HEAD(&new_driver->dynids.list);

        retval = driver_register(&new_driver->driver);
        if (retval)
                goto out;

        retval = usb_create_newid_files(new_driver);
        if (retval)
                goto out_newid;

        pr_info("%s: registered new interface driver %s\n",
                        usbcore_name, new_driver->name);

out:
        return retval;

out_newid:
        driver_unregister(&new_driver->driver);

        pr_err("%s: error %d registering interface driver %s\n",
                usbcore_name, retval, new_driver->name);
        goto out;
}
EXPORT_SYMBOL_GPL(usb_register_driver);

/**
 * usb_deregister - unregister a USB interface driver
 * @driver: USB operations of the interface driver to unregister
 * Context: must be able to sleep
 *
 * Unlinks the specified driver from the internal USB driver list.
 *
 * NOTE: If you called usb_register_dev(), you still need to call
 * usb_deregister_dev() to clean up your driver's allocated minor numbers,
 * this * call will no longer do it for you.
 */
void usb_deregister(struct usb_driver *driver)
{
        pr_info("%s: deregistering interface driver %s\n",
                        usbcore_name, driver->name);

        usb_remove_newid_files(driver);
        driver_unregister(&driver->driver);
        usb_free_dynids(driver);
}
EXPORT_SYMBOL_GPL(usb_deregister);

/* Forced unbinding of a USB interface driver, either because
 * it doesn't support pre_reset/post_reset/reset_resume or
 * because it doesn't support suspend/resume.
 *
 * The caller must hold @intf's device's lock, but not @intf's lock.
 */
void usb_forced_unbind_intf(struct usb_interface *intf)
{
        struct usb_driver *driver = to_usb_driver(intf->dev.driver);

        dev_dbg(&intf->dev, "forced unbind\n");
        usb_driver_release_interface(driver, intf);

        /* Mark the interface for later rebinding */
        intf->needs_binding = 1;
}

/*
 * Unbind drivers for @udev's marked interfaces.  These interfaces have
 * the needs_binding flag set, for example by usb_resume_interface().
 *
 * The caller must hold @udev's device lock.
 */
static void unbind_marked_interfaces(struct usb_device *udev)
{
        struct usb_host_config        *config;
        int                        i;
        struct usb_interface        *intf;

        config = udev->actconfig;
        if (config) {
                for (i = 0; i < config->desc.bNumInterfaces; ++i) {
                        intf = config->interface[i];
                        if (intf->dev.driver && intf->needs_binding)
                                usb_forced_unbind_intf(intf);
                }
        }
}

/* Delayed forced unbinding of a USB interface driver and scan
 * for rebinding.
 *
 * The caller must hold @intf's device's lock, but not @intf's lock.
 *
 * Note: Rebinds will be skipped if a system sleep transition is in
 * progress and the PM "complete" callback hasn't occurred yet.
 */
static void usb_rebind_intf(struct usb_interface *intf)
{
        int rc;

        /* Delayed unbind of an existing driver */
        if (intf->dev.driver)
                usb_forced_unbind_intf(intf);

        /* Try to rebind the interface */
        if (!intf->dev.power.is_prepared) {
                intf->needs_binding = 0;
                rc = device_attach(&intf->dev);
                if (rc < 0 && rc != -EPROBE_DEFER)
                        dev_warn(&intf->dev, "rebind failed: %d\n", rc);
        }
}

/*
 * Rebind drivers to @udev's marked interfaces.  These interfaces have
 * the needs_binding flag set.
 *
 * The caller must hold @udev's device lock.
 */
static void rebind_marked_interfaces(struct usb_device *udev)
{
        struct usb_host_config        *config;
        int                        i;
        struct usb_interface        *intf;

        config = udev->actconfig;
        if (config) {
                for (i = 0; i < config->desc.bNumInterfaces; ++i) {
                        intf = config->interface[i];
                        if (intf->needs_binding)
                                usb_rebind_intf(intf);
                }
        }
}

/*
 * Unbind all of @udev's marked interfaces and then rebind all of them.
 * This ordering is necessary because some drivers claim several interfaces
 * when they are first probed.
 *
 * The caller must hold @udev's device lock.
 */
void usb_unbind_and_rebind_marked_interfaces(struct usb_device *udev)
{
        unbind_marked_interfaces(udev);
        rebind_marked_interfaces(udev);
}

#ifdef CONFIG_PM

/* Unbind drivers for @udev's interfaces that don't support suspend/resume
 * There is no check for reset_resume here because it can be determined
 * only during resume whether reset_resume is needed.
 *
 * The caller must hold @udev's device lock.
 */
static void unbind_no_pm_drivers_interfaces(struct usb_device *udev)
{
        struct usb_host_config        *config;
        int                        i;
        struct usb_interface        *intf;
        struct usb_driver        *drv;

        config = udev->actconfig;
        if (config) {
                for (i = 0; i < config->desc.bNumInterfaces; ++i) {
                        intf = config->interface[i];

                        if (intf->dev.driver) {
                                drv = to_usb_driver(intf->dev.driver);
                                if (!drv->suspend || !drv->resume)
                                        usb_forced_unbind_intf(intf);
                        }
                }
        }
}

static int usb_suspend_device(struct usb_device *udev, pm_message_t msg)
{
        struct usb_device_driver        *udriver;
        int                                status = 0;

        if (udev->state == USB_STATE_NOTATTACHED ||
                        udev->state == USB_STATE_SUSPENDED)
                goto done;

        /* For devices that don't have a driver, we do a generic suspend. */
        if (udev->dev.driver)
                udriver = to_usb_device_driver(udev->dev.driver);
        else {
                udev->do_remote_wakeup = 0;
                udriver = &usb_generic_driver;
        }
        if (udriver->suspend)
                status = udriver->suspend(udev, msg);
        if (status == 0 && udriver->generic_subclass)
                status = usb_generic_driver_suspend(udev, msg);

 done:
        dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status);
        return status;
}

static int usb_resume_device(struct usb_device *udev, pm_message_t msg)
{
        struct usb_device_driver        *udriver;
        int                                status = 0;

        if (udev->state == USB_STATE_NOTATTACHED)
                goto done;

        /* Can't resume it if it doesn't have a driver. */
        if (udev->dev.driver == NULL) {
                status = -ENOTCONN;
                goto done;
        }

        /* Non-root devices on a full/low-speed bus must wait for their
         * companion high-speed root hub, in case a handoff is needed.
         */
        if (!PMSG_IS_AUTO(msg) && udev->parent && udev->bus->hs_companion)
                device_pm_wait_for_dev(&udev->dev,
                                &udev->bus->hs_companion->root_hub->dev);

        if (udev->quirks & USB_QUIRK_RESET_RESUME)
                udev->reset_resume = 1;

        udriver = to_usb_device_driver(udev->dev.driver);
        if (udriver->generic_subclass)
                status = usb_generic_driver_resume(udev, msg);
        if (status == 0 && udriver->resume)
                status = udriver->resume(udev, msg);

 done:
        dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status);
        return status;
}

static int usb_suspend_interface(struct usb_device *udev,
                struct usb_interface *intf, pm_message_t msg)
{
        struct usb_driver        *driver;
        int                        status = 0;

        if (udev->state == USB_STATE_NOTATTACHED ||
                        intf->condition == USB_INTERFACE_UNBOUND)
                goto done;
        driver = to_usb_driver(intf->dev.driver);

        /* at this time we know the driver supports suspend */
        status = driver->suspend(intf, msg);
        if (status && !PMSG_IS_AUTO(msg))
                dev_err(&intf->dev, "suspend error %d\n", status);

 done:
        dev_vdbg(&intf->dev, "%s: status %d\n", __func__, status);
        return status;
}

static int usb_resume_interface(struct usb_device *udev,
                struct usb_interface *intf, pm_message_t msg, int reset_resume)
{
        struct usb_driver        *driver;
        int                        status = 0;

        if (udev->state == USB_STATE_NOTATTACHED)
                goto done;

        /* Don't let autoresume interfere with unbinding */
        if (intf->condition == USB_INTERFACE_UNBINDING)
                goto done;

        /* Can't resume it if it doesn't have a driver. */
        if (intf->condition == USB_INTERFACE_UNBOUND) {

                /* Carry out a deferred switch to altsetting 0 */
                if (intf->needs_altsetting0 && !intf->dev.power.is_prepared) {
                        usb_set_interface(udev, intf->altsetting[0].
                                        desc.bInterfaceNumber, 0);
                        intf->needs_altsetting0 = 0;
                }
                goto done;
        }

        /* Don't resume if the interface is marked for rebinding */
        if (intf->needs_binding)
                goto done;
        driver = to_usb_driver(intf->dev.driver);

        if (reset_resume) {
                if (driver->reset_resume) {
                        status = driver->reset_resume(intf);
                        if (status)
                                dev_err(&intf->dev, "%s error %d\n",
                                                "reset_resume", status);
                } else {
                        intf->needs_binding = 1;
                        dev_dbg(&intf->dev, "no reset_resume for driver %s?\n",
                                        driver->name);
                }
        } else {
                status = driver->resume(intf);
                if (status)
                        dev_err(&intf->dev, "resume error %d\n", status);
        }

done:
        dev_vdbg(&intf->dev, "%s: status %d\n", __func__, status);

        /* Later we will unbind the driver and/or reprobe, if necessary */
        return status;
}

/**
 * usb_suspend_both - suspend a USB device and its interfaces
 * @udev: the usb_device to suspend
 * @msg: Power Management message describing this state transition
 *
 * This is the central routine for suspending USB devices.  It calls the
 * suspend methods for all the interface drivers in @udev and then calls
 * the suspend method for @udev itself.  When the routine is called in
 * autosuspend, if an error occurs at any stage, all the interfaces
 * which were suspended are resumed so that they remain in the same
 * state as the device, but when called from system sleep, all error
 * from suspend methods of interfaces and the non-root-hub device itself
 * are simply ignored, so all suspended interfaces are only resumed
 * to the device's state when @udev is root-hub and its suspend method
 * returns failure.
 *
 * Autosuspend requests originating from a child device or an interface
 * driver may be made without the protection of @udev's device lock, but
 * all other suspend calls will hold the lock.  Usbcore will insure that
 * method calls do not arrive during bind, unbind, or reset operations.
 * However drivers must be prepared to handle suspend calls arriving at
 * unpredictable times.
 *
 * This routine can run only in process context.
 *
 * Return: 0 if the suspend succeeded.
 */
static int usb_suspend_both(struct usb_device *udev, pm_message_t msg)
{
        int                        status = 0;
        int                        i = 0, n = 0;
        struct usb_interface        *intf;

        if (udev->state == USB_STATE_NOTATTACHED ||
                        udev->state == USB_STATE_SUSPENDED)
                goto done;

        /* Suspend all the interfaces and then udev itself */
        if (udev->actconfig) {
                n = udev->actconfig->desc.bNumInterfaces;
                for (i = n - 1; i >= 0; --i) {
                        intf = udev->actconfig->interface[i];
                        status = usb_suspend_interface(udev, intf, msg);

                        /* Ignore errors during system sleep transitions */
                        if (!PMSG_IS_AUTO(msg))
                                status = 0;
                        if (status != 0)
                                break;
                }
        }
        if (status == 0) {
                status = usb_suspend_device(udev, msg);

                /*
                 * Ignore errors from non-root-hub devices during
                 * system sleep transitions.  For the most part,
                 * these devices should go to low power anyway when
                 * the entire bus is suspended.
                 */
                if (udev->parent && !PMSG_IS_AUTO(msg))
                        status = 0;

                /*
                 * If the device is inaccessible, don't try to resume
                 * suspended interfaces and just return the error.
                 */
                if (status && status != -EBUSY) {
                        int err;
                        u16 devstat;

                        err = usb_get_std_status(udev, USB_RECIP_DEVICE, 0,
                                                 &devstat);
                        if (err) {
                                dev_err(&udev->dev,
                                        "Failed to suspend device, error %d\n",
                                        status);
                                goto done;
                        }
                }
        }

        /* If the suspend failed, resume interfaces that did get suspended */
        if (status != 0) {
                if (udev->actconfig) {
                        msg.event ^= (PM_EVENT_SUSPEND | PM_EVENT_RESUME);
                        while (++i < n) {
                                intf = udev->actconfig->interface[i];
                                usb_resume_interface(udev, intf, msg, 0);
                        }
                }

        /* If the suspend succeeded then prevent any more URB submissions
         * and flush any outstanding URBs.
         */
        } else {
                udev->can_submit = 0;
                for (i = 0; i < 16; ++i) {
                        usb_hcd_flush_endpoint(udev, udev->ep_out[i]);
                        usb_hcd_flush_endpoint(udev, udev->ep_in[i]);
                }
        }

 done:
        dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status);
        return status;
}

/**
 * usb_resume_both - resume a USB device and its interfaces
 * @udev: the usb_device to resume
 * @msg: Power Management message describing this state transition
 *
 * This is the central routine for resuming USB devices.  It calls the
 * resume method for @udev and then calls the resume methods for all
 * the interface drivers in @udev.
 *
 * Autoresume requests originating from a child device or an interface
 * driver may be made without the protection of @udev's device lock, but
 * all other resume calls will hold the lock.  Usbcore will insure that
 * method calls do not arrive during bind, unbind, or reset operations.
 * However drivers must be prepared to handle resume calls arriving at
 * unpredictable times.
 *
 * This routine can run only in process context.
 *
 * Return: 0 on success.
 */
static int usb_resume_both(struct usb_device *udev, pm_message_t msg)
{
        int                        status = 0;
        int                        i;
        struct usb_interface        *intf;

        if (udev->state == USB_STATE_NOTATTACHED) {
                status = -ENODEV;
                goto done;
        }
        udev->can_submit = 1;

        /* Resume the device */
        if (udev->state == USB_STATE_SUSPENDED || udev->reset_resume)
                status = usb_resume_device(udev, msg);

        /* Resume the interfaces */
        if (status == 0 && udev->actconfig) {
                for (i = 0; i < udev->actconfig->desc.bNumInterfaces; i++) {
                        intf = udev->actconfig->interface[i];
                        usb_resume_interface(udev, intf, msg,
                                        udev->reset_resume);
                }
        }
        usb_mark_last_busy(udev);

 done:
        dev_vdbg(&udev->dev, "%s: status %d\n", __func__, status);
        if (!status)
                udev->reset_resume = 0;
        return status;
}

static void choose_wakeup(struct usb_device *udev, pm_message_t msg)
{
        int        w;

        /*
         * For FREEZE/QUIESCE, disable remote wakeups so no interrupts get
         * generated.
         */
        if (msg.event == PM_EVENT_FREEZE || msg.event == PM_EVENT_QUIESCE) {
                w = 0;

        } else {
                /*
                 * Enable remote wakeup if it is allowed, even if no interface
                 * drivers actually want it.
                 */
                w = device_may_wakeup(&udev->dev);
        }

        /*
         * If the device is autosuspended with the wrong wakeup setting,
         * autoresume now so the setting can be changed.
         */
        if (udev->state == USB_STATE_SUSPENDED && w != udev->do_remote_wakeup)
                pm_runtime_resume(&udev->dev);
        udev->do_remote_wakeup = w;
}

/* The device lock is held by the PM core */
int usb_suspend(struct device *dev, pm_message_t msg)
{
        struct usb_device        *udev = to_usb_device(dev);
        int r;

        unbind_no_pm_drivers_interfaces(udev);

        /* From now on we are sure all drivers support suspend/resume
         * but not necessarily reset_resume()
         * so we may still need to unbind and rebind upon resume
         */
        choose_wakeup(udev, msg);
        r = usb_suspend_both(udev, msg);
        if (r)
                return r;

        if (udev->quirks & USB_QUIRK_DISCONNECT_SUSPEND)
                usb_port_disable(udev);

        return 0;
}

/* The device lock is held by the PM core */
int usb_resume_complete(struct device *dev)
{
        struct usb_device *udev = to_usb_device(dev);

        /* For PM complete calls, all we do is rebind interfaces
         * whose needs_binding flag is set
         */
        if (udev->state != USB_STATE_NOTATTACHED)
                rebind_marked_interfaces(udev);
        return 0;
}

/* The device lock is held by the PM core */
int usb_resume(struct device *dev, pm_message_t msg)
{
        struct usb_device        *udev = to_usb_device(dev);
        int                        status;

        /* For all calls, take the device back to full power and
         * tell the PM core in case it was autosuspended previously.
         * Unbind the interfaces that will need rebinding later,
         * because they fail to support reset_resume.
         * (This can't be done in usb_resume_interface()
         * above because it doesn't own the right set of locks.)
         */
        status = usb_resume_both(udev, msg);
        if (status == 0) {
                pm_runtime_disable(dev);
                pm_runtime_set_active(dev);
                pm_runtime_enable(dev);
                unbind_marked_interfaces(udev);
        }

        /* Avoid PM error messages for devices disconnected while suspended
         * as we'll display regular disconnect messages just a bit later.
         */
        if (status == -ENODEV || status == -ESHUTDOWN)
                status = 0;
        return status;
}

/**
 * usb_enable_autosuspend - allow a USB device to be autosuspended
 * @udev: the USB device which may be autosuspended
 *
 * This routine allows @udev to be autosuspended.  An autosuspend won't
 * take place until the autosuspend_delay has elapsed and all the other
 * necessary conditions are satisfied.
 *
 * The caller must hold @udev's device lock.
 */
void usb_enable_autosuspend(struct usb_device *udev)
{
        pm_runtime_allow(&udev->dev);
}
EXPORT_SYMBOL_GPL(usb_enable_autosuspend);

/**
 * usb_disable_autosuspend - prevent a USB device from being autosuspended
 * @udev: the USB device which may not be autosuspended
 *
 * This routine prevents @udev from being autosuspended and wakes it up
 * if it is already autosuspended.
 *
 * The caller must hold @udev's device lock.
 */
void usb_disable_autosuspend(struct usb_device *udev)
{
        pm_runtime_forbid(&udev->dev);
}
EXPORT_SYMBOL_GPL(usb_disable_autosuspend);

/**
 * usb_autosuspend_device - delayed autosuspend of a USB device and its interfaces
 * @udev: the usb_device to autosuspend
 *
 * This routine should be called when a core subsystem is finished using
 * @udev and wants to allow it to autosuspend.  Examples would be when
 * @udev's device file in usbfs is closed or after a configuration change.
 *
 * @udev's usage counter is decremented; if it drops to 0 and all the
 * interfaces are inactive then a delayed autosuspend will be attempted.
 * The attempt may fail (see autosuspend_check()).
 *
 * The caller must hold @udev's device lock.
 *
 * This routine can run only in process context.
 */
void usb_autosuspend_device(struct usb_device *udev)
{
        int        status;

        usb_mark_last_busy(udev);
        status = pm_runtime_put_sync_autosuspend(&udev->dev);
        dev_vdbg(&udev->dev, "%s: cnt %d -> %d\n",
                        __func__, atomic_read(&udev->dev.power.usage_count),
                        status);
}

/**
 * usb_autoresume_device - immediately autoresume a USB device and its interfaces
 * @udev: the usb_device to autoresume
 *
 * This routine should be called when a core subsystem wants to use @udev
 * and needs to guarantee that it is not suspended.  No autosuspend will
 * occur until usb_autosuspend_device() is called.  (Note that this will
 * not prevent suspend events originating in the PM core.)  Examples would
 * be when @udev's device file in usbfs is opened or when a remote-wakeup
 * request is received.
 *
 * @udev's usage counter is incremented to prevent subsequent autosuspends.
 * However if the autoresume fails then the usage counter is re-decremented.
 *
 * The caller must hold @udev's device lock.
 *
 * This routine can run only in process context.
 *
 * Return: 0 on success. A negative error code otherwise.
 */
int usb_autoresume_device(struct usb_device *udev)
{
        int        status;

        status = pm_runtime_resume_and_get(&udev->dev);
        dev_vdbg(&udev->dev, "%s: cnt %d -> %d\n",
                        __func__, atomic_read(&udev->dev.power.usage_count),
                        status);
        if (status > 0)
                status = 0;
        return status;
}

/**
 * usb_autopm_put_interface - decrement a USB interface's PM-usage counter
 * @intf: the usb_interface whose counter should be decremented
 *
 * This routine should be called by an interface driver when it is
 * finished using @intf and wants to allow it to autosuspend.  A typical
 * example would be a character-device driver when its device file is
 * closed.
 *
 * The routine decrements @intf's usage counter.  When the counter reaches
 * 0, a delayed autosuspend request for @intf's device is attempted.  The
 * attempt may fail (see autosuspend_check()).
 *
 * This routine can run only in process context.
 */
void usb_autopm_put_interface(struct usb_interface *intf)
{
        struct usb_device        *udev = interface_to_usbdev(intf);
        int                        status;

        usb_mark_last_busy(udev);
        status = pm_runtime_put_sync(&intf->dev);
        dev_vdbg(&intf->dev, "%s: cnt %d -> %d\n",
                        __func__, atomic_read(&intf->dev.power.usage_count),
                        status);
}
EXPORT_SYMBOL_GPL(usb_autopm_put_interface);

/**
 * usb_autopm_put_interface_async - decrement a USB interface's PM-usage counter
 * @intf: the usb_interface whose counter should be decremented
 *
 * This routine does much the same thing as usb_autopm_put_interface():
 * It decrements @intf's usage counter and schedules a delayed
 * autosuspend request if the counter is <= 0.  The difference is that it
 * does not perform any synchronization; callers should hold a private
 * lock and handle all synchronization issues themselves.
 *
 * Typically a driver would call this routine during an URB's completion
 * handler, if no more URBs were pending.
 *
 * This routine can run in atomic context.
 */
void usb_autopm_put_interface_async(struct usb_interface *intf)
{
        struct usb_device        *udev = interface_to_usbdev(intf);
        int                        status;

        usb_mark_last_busy(udev);
        status = pm_runtime_put(&intf->dev);
        dev_vdbg(&intf->dev, "%s: cnt %d -> %d\n",
                        __func__, atomic_read(&intf->dev.power.usage_count),
                        status);
}
EXPORT_SYMBOL_GPL(usb_autopm_put_interface_async);

/**
 * usb_autopm_put_interface_no_suspend - decrement a USB interface's PM-usage counter
 * @intf: the usb_interface whose counter should be decremented
 *
 * This routine decrements @intf's usage counter but does not carry out an
 * autosuspend.
 *
 * This routine can run in atomic context.
 */
void usb_autopm_put_interface_no_suspend(struct usb_interface *intf)
{
        struct usb_device        *udev = interface_to_usbdev(intf);

        usb_mark_last_busy(udev);
        pm_runtime_put_noidle(&intf->dev);
}
EXPORT_SYMBOL_GPL(usb_autopm_put_interface_no_suspend);

/**
 * usb_autopm_get_interface - increment a USB interface's PM-usage counter
 * @intf: the usb_interface whose counter should be incremented
 *
 * This routine should be called by an interface driver when it wants to
 * use @intf and needs to guarantee that it is not suspended.  In addition,
 * the routine prevents @intf from being autosuspended subsequently.  (Note
 * that this will not prevent suspend events originating in the PM core.)
 * This prevention will persist until usb_autopm_put_interface() is called
 * or @intf is unbound.  A typical example would be a character-device
 * driver when its device file is opened.
 *
 * @intf's usage counter is incremented to prevent subsequent autosuspends.
 * However if the autoresume fails then the counter is re-decremented.
 *
 * This routine can run only in process context.
 *
 * Return: 0 on success.
 */
int usb_autopm_get_interface(struct usb_interface *intf)
{
        int        status;

        status = pm_runtime_resume_and_get(&intf->dev);
        dev_vdbg(&intf->dev, "%s: cnt %d -> %d\n",
                        __func__, atomic_read(&intf->dev.power.usage_count),
                        status);
        if (status > 0)
                status = 0;
        return status;
}
EXPORT_SYMBOL_GPL(usb_autopm_get_interface);

/**
 * usb_autopm_get_interface_async - increment a USB interface's PM-usage counter
 * @intf: the usb_interface whose counter should be incremented
 *
 * This routine does much the same thing as
 * usb_autopm_get_interface(): It increments @intf's usage counter and
 * queues an autoresume request if the device is suspended.  The
 * differences are that it does not perform any synchronization (callers
 * should hold a private lock and handle all synchronization issues
 * themselves), and it does not autoresume the device directly (it only
 * queues a request).  After a successful call, the device may not yet be
 * resumed.
 *
 * This routine can run in atomic context.
 *
 * Return: 0 on success. A negative error code otherwise.
 */
int usb_autopm_get_interface_async(struct usb_interface *intf)
{
        int        status;

        status = pm_runtime_get(&intf->dev);
        if (status < 0 && status != -EINPROGRESS)
                pm_runtime_put_noidle(&intf->dev);
        dev_vdbg(&intf->dev, "%s: cnt %d -> %d\n",
                        __func__, atomic_read(&intf->dev.power.usage_count),
                        status);
        if (status > 0 || status == -EINPROGRESS)
                status = 0;
        return status;
}
EXPORT_SYMBOL_GPL(usb_autopm_get_interface_async);

/**
 * usb_autopm_get_interface_no_resume - increment a USB interface's PM-usage counter
 * @intf: the usb_interface whose counter should be incremented
 *
 * This routine increments @intf's usage counter but does not carry out an
 * autoresume.
 *
 * This routine can run in atomic context.
 */
void usb_autopm_get_interface_no_resume(struct usb_interface *intf)
{
        struct usb_device        *udev = interface_to_usbdev(intf);

        usb_mark_last_busy(udev);
        pm_runtime_get_noresume(&intf->dev);
}
EXPORT_SYMBOL_GPL(usb_autopm_get_interface_no_resume);

/* Internal routine to check whether we may autosuspend a device. */
static int autosuspend_check(struct usb_device *udev)
{
        int                        w, i;
        struct usb_interface        *intf;

        if (udev->state == USB_STATE_NOTATTACHED)
                return -ENODEV;

        /* Fail if autosuspend is disabled, or any interfaces are in use, or
         * any interface drivers require remote wakeup but it isn't available.
         */
        w = 0;
        if (udev->actconfig) {
                for (i = 0; i < udev->actconfig->desc.bNumInterfaces; i++) {
                        intf = udev->actconfig->interface[i];

                        /* We don't need to check interfaces that are
                         * disabled for runtime PM.  Either they are unbound
                         * or else their drivers don't support autosuspend
                         * and so they are permanently active.
                         */
                        if (intf->dev.power.disable_depth)
                                continue;
                        if (atomic_read(&intf->dev.power.usage_count) > 0)
                                return -EBUSY;
                        w |= intf->needs_remote_wakeup;

                        /* Don't allow autosuspend if the device will need
                         * a reset-resume and any of its interface drivers
                         * doesn't include support or needs remote wakeup.
                         */
                        if (udev->quirks & USB_QUIRK_RESET_RESUME) {
                                struct usb_driver *driver;

                                driver = to_usb_driver(intf->dev.driver);
                                if (!driver->reset_resume ||
                                                intf->needs_remote_wakeup)
                                        return -EOPNOTSUPP;
                        }
                }
        }
        if (w && !device_can_wakeup(&udev->dev)) {
                dev_dbg(&udev->dev, "remote wakeup needed for autosuspend\n");
                return -EOPNOTSUPP;
        }

        /*
         * If the device is a direct child of the root hub and the HCD
         * doesn't handle wakeup requests, don't allow autosuspend when
         * wakeup is needed.
         */
        if (w && udev->parent == udev->bus->root_hub &&
                        bus_to_hcd(udev->bus)->cant_recv_wakeups) {
                dev_dbg(&udev->dev, "HCD doesn't handle wakeup requests\n");
                return -EOPNOTSUPP;
        }

        udev->do_remote_wakeup = w;
        return 0;
}

int usb_runtime_suspend(struct device *dev)
{
        struct usb_device        *udev = to_usb_device(dev);
        int                        status;

        /* A USB device can be suspended if it passes the various autosuspend
         * checks.  Runtime suspend for a USB device means suspending all the
         * interfaces and then the device itself.
         */
        if (autosuspend_check(udev) != 0)
                return -EAGAIN;

        status = usb_suspend_both(udev, PMSG_AUTO_SUSPEND);

        /* Allow a retry if autosuspend failed temporarily */
        if (status == -EAGAIN || status == -EBUSY)
                usb_mark_last_busy(udev);

        /*
         * The PM core reacts badly unless the return code is 0,
         * -EAGAIN, or -EBUSY, so always return -EBUSY on an error
         * (except for root hubs, because they don't suspend through
         * an upstream port like other USB devices).
         */
        if (status != 0 && udev->parent)
                return -EBUSY;
        return status;
}

int usb_runtime_resume(struct device *dev)
{
        struct usb_device        *udev = to_usb_device(dev);
        int                        status;

        /* Runtime resume for a USB device means resuming both the device
         * and all its interfaces.
         */
        status = usb_resume_both(udev, PMSG_AUTO_RESUME);
        return status;
}

int usb_runtime_idle(struct device *dev)
{
        struct usb_device        *udev = to_usb_device(dev);

        /* An idle USB device can be suspended if it passes the various
         * autosuspend checks.
         */
        if (autosuspend_check(udev) == 0)
                pm_runtime_autosuspend(dev);
        /* Tell the core not to suspend it, though. */
        return -EBUSY;
}

static int usb_set_usb2_hardware_lpm(struct usb_device *udev, int enable)
{
        struct usb_hcd *hcd = bus_to_hcd(udev->bus);
        int ret = -EPERM;

        if (hcd->driver->set_usb2_hw_lpm) {
                ret = hcd->driver->set_usb2_hw_lpm(hcd, udev, enable);
                if (!ret)
                        udev->usb2_hw_lpm_enabled = enable;
        }

        return ret;
}

int usb_enable_usb2_hardware_lpm(struct usb_device *udev)
{
        if (!udev->usb2_hw_lpm_capable ||
            !udev->usb2_hw_lpm_allowed ||
            udev->usb2_hw_lpm_enabled)
                return 0;

        return usb_set_usb2_hardware_lpm(udev, 1);
}

int usb_disable_usb2_hardware_lpm(struct usb_device *udev)
{
        if (!udev->usb2_hw_lpm_enabled)
                return 0;

        return usb_set_usb2_hardware_lpm(udev, 0);
}

#endif /* CONFIG_PM */

const struct bus_type usb_bus_type = {
        .name =                "usb",
        .match =        usb_device_match,
        .uevent =        usb_uevent,
        .need_parent_lock =        true,
};













































































































































































































































































   32 
   35 
















   56 
   56 






    7 






















   49 


   56 
   56 














   10 







    2 



    2 






    2 



    2 







   36 







    6 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Landlock LSM - Ruleset management
 *
 * Copyright © 2016-2020 Mickaël Salaün <mic@digikod.net>
 * Copyright © 2018-2020 ANSSI
 */

#ifndef _SECURITY_LANDLOCK_RULESET_H
#define _SECURITY_LANDLOCK_RULESET_H

#include <linux/bitops.h>
#include <linux/build_bug.h>
#include <linux/kernel.h>
#include <linux/mutex.h>
#include <linux/rbtree.h>
#include <linux/refcount.h>
#include <linux/workqueue.h>
#include <uapi/linux/landlock.h>

#include "limits.h"
#include "object.h"

/*
 * All access rights that are denied by default whether they are handled or not
 * by a ruleset/layer.  This must be ORed with all ruleset->access_masks[]
 * entries when we need to get the absolute handled access masks.
 */
/* clang-format off */
#define LANDLOCK_ACCESS_FS_INITIALLY_DENIED ( \
        LANDLOCK_ACCESS_FS_REFER)
/* clang-format on */

typedef u16 access_mask_t;
/* Makes sure all filesystem access rights can be stored. */
static_assert(BITS_PER_TYPE(access_mask_t) >= LANDLOCK_NUM_ACCESS_FS);
/* Makes sure all network access rights can be stored. */
static_assert(BITS_PER_TYPE(access_mask_t) >= LANDLOCK_NUM_ACCESS_NET);
/* Makes sure all scoped rights can be stored. */
static_assert(BITS_PER_TYPE(access_mask_t) >= LANDLOCK_NUM_SCOPE);
/* Makes sure for_each_set_bit() and for_each_clear_bit() calls are OK. */
static_assert(sizeof(unsigned long) >= sizeof(access_mask_t));

/* Ruleset access masks. */
struct access_masks {
        access_mask_t fs : LANDLOCK_NUM_ACCESS_FS;
        access_mask_t net : LANDLOCK_NUM_ACCESS_NET;
        access_mask_t scope : LANDLOCK_NUM_SCOPE;
};

union access_masks_all {
        struct access_masks masks;
        u32 all;
};

/* Makes sure all fields are covered. */
static_assert(sizeof(typeof_member(union access_masks_all, masks)) ==
              sizeof(typeof_member(union access_masks_all, all)));

typedef u16 layer_mask_t;
/* Makes sure all layers can be checked. */
static_assert(BITS_PER_TYPE(layer_mask_t) >= LANDLOCK_MAX_NUM_LAYERS);

/**
 * struct landlock_layer - Access rights for a given layer
 */
struct landlock_layer {
        /**
         * @level: Position of this layer in the layer stack.
         */
        u16 level;
        /**
         * @access: Bitfield of allowed actions on the kernel object.  They are
         * relative to the object type (e.g. %LANDLOCK_ACTION_FS_READ).
         */
        access_mask_t access;
};

/**
 * union landlock_key - Key of a ruleset's red-black tree
 */
union landlock_key {
        /**
         * @object: Pointer to identify a kernel object (e.g. an inode).
         */
        struct landlock_object *object;
        /**
         * @data: Raw data to identify an arbitrary 32-bit value
         * (e.g. a TCP port).
         */
        uintptr_t data;
};

/**
 * enum landlock_key_type - Type of &union landlock_key
 */
enum landlock_key_type {
        /**
         * @LANDLOCK_KEY_INODE: Type of &landlock_ruleset.root_inode's node
         * keys.
         */
        LANDLOCK_KEY_INODE = 1,
        /**
         * @LANDLOCK_KEY_NET_PORT: Type of &landlock_ruleset.root_net_port's
         * node keys.
         */
        LANDLOCK_KEY_NET_PORT,
};

/**
 * struct landlock_id - Unique rule identifier for a ruleset
 */
struct landlock_id {
        /**
         * @key: Identifies either a kernel object (e.g. an inode) or
         * a raw value (e.g. a TCP port).
         */
        union landlock_key key;
        /**
         * @type: Type of a landlock_ruleset's root tree.
         */
        const enum landlock_key_type type;
};

/**
 * struct landlock_rule - Access rights tied to an object
 */
struct landlock_rule {
        /**
         * @node: Node in the ruleset's red-black tree.
         */
        struct rb_node node;
        /**
         * @key: A union to identify either a kernel object (e.g. an inode) or
         * a raw data value (e.g. a network socket port). This is used as a key
         * for this ruleset element.  The pointer is set once and never
         * modified.  It always points to an allocated object because each rule
         * increments the refcount of its object.
         */
        union landlock_key key;
        /**
         * @num_layers: Number of entries in @layers.
         */
        u32 num_layers;
        /**
         * @layers: Stack of layers, from the latest to the newest, implemented
         * as a flexible array member (FAM).
         */
        struct landlock_layer layers[] __counted_by(num_layers);
};

/**
 * struct landlock_hierarchy - Node in a ruleset hierarchy
 */
struct landlock_hierarchy {
        /**
         * @parent: Pointer to the parent node, or NULL if it is a root
         * Landlock domain.
         */
        struct landlock_hierarchy *parent;
        /**
         * @usage: Number of potential children domains plus their parent
         * domain.
         */
        refcount_t usage;
};

/**
 * struct landlock_ruleset - Landlock ruleset
 *
 * This data structure must contain unique entries, be updatable, and quick to
 * match an object.
 */
struct landlock_ruleset {
        /**
         * @root_inode: Root of a red-black tree containing &struct
         * landlock_rule nodes with inode object.  Once a ruleset is tied to a
         * process (i.e. as a domain), this tree is immutable until @usage
         * reaches zero.
         */
        struct rb_root root_inode;

#if IS_ENABLED(CONFIG_INET)
        /**
         * @root_net_port: Root of a red-black tree containing &struct
         * landlock_rule nodes with network port. Once a ruleset is tied to a
         * process (i.e. as a domain), this tree is immutable until @usage
         * reaches zero.
         */
        struct rb_root root_net_port;
#endif /* IS_ENABLED(CONFIG_INET) */

        /**
         * @hierarchy: Enables hierarchy identification even when a parent
         * domain vanishes.  This is needed for the ptrace protection.
         */
        struct landlock_hierarchy *hierarchy;
        union {
                /**
                 * @work_free: Enables to free a ruleset within a lockless
                 * section.  This is only used by
                 * landlock_put_ruleset_deferred() when @usage reaches zero.
                 * The fields @lock, @usage, @num_rules, @num_layers and
                 * @access_masks are then unused.
                 */
                struct work_struct work_free;
                struct {
                        /**
                         * @lock: Protects against concurrent modifications of
                         * @root, if @usage is greater than zero.
                         */
                        struct mutex lock;
                        /**
                         * @usage: Number of processes (i.e. domains) or file
                         * descriptors referencing this ruleset.
                         */
                        refcount_t usage;
                        /**
                         * @num_rules: Number of non-overlapping (i.e. not for
                         * the same object) rules in this ruleset.
                         */
                        u32 num_rules;
                        /**
                         * @num_layers: Number of layers that are used in this
                         * ruleset.  This enables to check that all the layers
                         * allow an access request.  A value of 0 identifies a
                         * non-merged ruleset (i.e. not a domain).
                         */
                        u32 num_layers;
                        /**
                         * @access_masks: Contains the subset of filesystem and
                         * network actions that are restricted by a ruleset.
                         * A domain saves all layers of merged rulesets in a
                         * stack (FAM), starting from the first layer to the
                         * last one.  These layers are used when merging
                         * rulesets, for user space backward compatibility
                         * (i.e. future-proof), and to properly handle merged
                         * rulesets without overlapping access rights.  These
                         * layers are set once and never changed for the
                         * lifetime of the ruleset.
                         */
                        struct access_masks access_masks[];
                };
        };
};

struct landlock_ruleset *
landlock_create_ruleset(const access_mask_t access_mask_fs,
                        const access_mask_t access_mask_net,
                        const access_mask_t scope_mask);

void landlock_put_ruleset(struct landlock_ruleset *const ruleset);
void landlock_put_ruleset_deferred(struct landlock_ruleset *const ruleset);

int landlock_insert_rule(struct landlock_ruleset *const ruleset,
                         const struct landlock_id id,
                         const access_mask_t access);

struct landlock_ruleset *
landlock_merge_ruleset(struct landlock_ruleset *const parent,
                       struct landlock_ruleset *const ruleset);

const struct landlock_rule *
landlock_find_rule(const struct landlock_ruleset *const ruleset,
                   const struct landlock_id id);

static inline void landlock_get_ruleset(struct landlock_ruleset *const ruleset)
{
        if (ruleset)
                refcount_inc(&ruleset->usage);
}

/**
 * landlock_union_access_masks - Return all access rights handled in the
 *                                 domain
 *
 * @domain: Landlock ruleset (used as a domain)
 *
 * Returns: an access_masks result of the OR of all the domain's access masks.
 */
static inline struct access_masks
landlock_union_access_masks(const struct landlock_ruleset *const domain)
{
        union access_masks_all matches = {};
        size_t layer_level;

        for (layer_level = 0; layer_level < domain->num_layers; layer_level++) {
                union access_masks_all layer = {
                        .masks = domain->access_masks[layer_level],
                };

                matches.all |= layer.all;
        }

        return matches.masks;
}

/**
 * landlock_get_applicable_domain - Return @domain if it applies to (handles)
 *                                    at least one of the access rights specified
 *                                    in @masks
 *
 * @domain: Landlock ruleset (used as a domain)
 * @masks: access masks
 *
 * Returns: @domain if any access rights specified in @masks is handled, or
 * NULL otherwise.
 */
static inline const struct landlock_ruleset *
landlock_get_applicable_domain(const struct landlock_ruleset *const domain,
                               const struct access_masks masks)
{
        const union access_masks_all masks_all = {
                .masks = masks,
        };
        union access_masks_all merge = {};

        if (!domain)
                return NULL;

        merge.masks = landlock_union_access_masks(domain);
        if (merge.all & masks_all.all)
                return domain;

        return NULL;
}

static inline void
landlock_add_fs_access_mask(struct landlock_ruleset *const ruleset,
                            const access_mask_t fs_access_mask,
                            const u16 layer_level)
{
        access_mask_t fs_mask = fs_access_mask & LANDLOCK_MASK_ACCESS_FS;

        /* Should already be checked in sys_landlock_create_ruleset(). */
        WARN_ON_ONCE(fs_access_mask != fs_mask);
        ruleset->access_masks[layer_level].fs |= fs_mask;
}

static inline void
landlock_add_net_access_mask(struct landlock_ruleset *const ruleset,
                             const access_mask_t net_access_mask,
                             const u16 layer_level)
{
        access_mask_t net_mask = net_access_mask & LANDLOCK_MASK_ACCESS_NET;

        /* Should already be checked in sys_landlock_create_ruleset(). */
        WARN_ON_ONCE(net_access_mask != net_mask);
        ruleset->access_masks[layer_level].net |= net_mask;
}

static inline void
landlock_add_scope_mask(struct landlock_ruleset *const ruleset,
                        const access_mask_t scope_mask, const u16 layer_level)
{
        access_mask_t mask = scope_mask & LANDLOCK_MASK_SCOPE;

        /* Should already be checked in sys_landlock_create_ruleset(). */
        WARN_ON_ONCE(scope_mask != mask);
        ruleset->access_masks[layer_level].scope |= mask;
}

static inline access_mask_t
landlock_get_fs_access_mask(const struct landlock_ruleset *const ruleset,
                            const u16 layer_level)
{
        /* Handles all initially denied by default access rights. */
        return ruleset->access_masks[layer_level].fs |
               LANDLOCK_ACCESS_FS_INITIALLY_DENIED;
}

static inline access_mask_t
landlock_get_net_access_mask(const struct landlock_ruleset *const ruleset,
                             const u16 layer_level)
{
        return ruleset->access_masks[layer_level].net;
}

static inline access_mask_t
landlock_get_scope_mask(const struct landlock_ruleset *const ruleset,
                        const u16 layer_level)
{
        return ruleset->access_masks[layer_level].scope;
}

bool landlock_unmask_layers(const struct landlock_rule *const rule,
                            const access_mask_t access_request,
                            layer_mask_t (*const layer_masks)[],
                            const size_t masks_array_size);

access_mask_t
landlock_init_layer_masks(const struct landlock_ruleset *const domain,
                          const access_mask_t access_request,
                          layer_mask_t (*const layer_masks)[],
                          const enum landlock_key_type key_type);

#endif /* _SECURITY_LANDLOCK_RULESET_H */
























































    2 








    2 






















































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  inode.c - securityfs
 *
 *  Copyright (C) 2005 Greg Kroah-Hartman <gregkh@suse.de>
 *
 *  Based on fs/debugfs/inode.c which had the following copyright notice:
 *    Copyright (C) 2004 Greg Kroah-Hartman <greg@kroah.com>
 *    Copyright (C) 2004 IBM Inc.
 */

/* #define DEBUG */
#include <linux/sysfs.h>
#include <linux/kobject.h>
#include <linux/fs.h>
#include <linux/fs_context.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/namei.h>
#include <linux/security.h>
#include <linux/lsm_hooks.h>
#include <linux/magic.h>

static struct vfsmount *mount;
static int mount_count;

static void securityfs_free_inode(struct inode *inode)
{
        if (S_ISLNK(inode->i_mode))
                kfree(inode->i_link);
        free_inode_nonrcu(inode);
}

static const struct super_operations securityfs_super_operations = {
        .statfs                = simple_statfs,
        .free_inode        = securityfs_free_inode,
};

static int securityfs_fill_super(struct super_block *sb, struct fs_context *fc)
{
        static const struct tree_descr files[] = {{""}};
        int error;

        error = simple_fill_super(sb, SECURITYFS_MAGIC, files);
        if (error)
                return error;

        sb->s_op = &securityfs_super_operations;

        return 0;
}

static int securityfs_get_tree(struct fs_context *fc)
{
        return get_tree_single(fc, securityfs_fill_super);
}

static const struct fs_context_operations securityfs_context_ops = {
        .get_tree        = securityfs_get_tree,
};

static int securityfs_init_fs_context(struct fs_context *fc)
{
        fc->ops = &securityfs_context_ops;
        return 0;
}

static struct file_system_type fs_type = {
        .owner =        THIS_MODULE,
        .name =                "securityfs",
        .init_fs_context = securityfs_init_fs_context,
        .kill_sb =        kill_litter_super,
};

/**
 * securityfs_create_dentry - create a dentry in the securityfs filesystem
 *
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the securityfs filesystem.
 * @data: a pointer to something that the caller will want to get to later
 *        on.  The inode.i_private pointer will point to this value on
 *        the open() call.
 * @fops: a pointer to a struct file_operations that should be used for
 *        this file.
 * @iops: a point to a struct of inode_operations that should be used for
 *        this file/dir
 *
 * This is the basic "create a file/dir/symlink" function for
 * securityfs.  It allows for a wide range of flexibility in creating
 * a file, or a directory (if you want to create a directory, the
 * securityfs_create_dir() function is recommended to be used
 * instead).
 *
 * This function returns a pointer to a dentry if it succeeds.  This
 * pointer must be passed to the securityfs_remove() function when the
 * file is to be removed (no automatic cleanup happens if your module
 * is unloaded, you are responsible here).  If an error occurs, the
 * function will return the error value (via ERR_PTR).
 *
 * If securityfs is not enabled in the kernel, the value %-ENODEV is
 * returned.
 */
static struct dentry *securityfs_create_dentry(const char *name, umode_t mode,
                                        struct dentry *parent, void *data,
                                        const struct file_operations *fops,
                                        const struct inode_operations *iops)
{
        struct dentry *dentry;
        struct inode *dir, *inode;
        int error;

        if (!(mode & S_IFMT))
                mode = (mode & S_IALLUGO) | S_IFREG;

        pr_debug("securityfs: creating file '%s'\n",name);

        error = simple_pin_fs(&fs_type, &mount, &mount_count);
        if (error)
                return ERR_PTR(error);

        if (!parent)
                parent = mount->mnt_root;

        dir = d_inode(parent);

        inode_lock(dir);
        dentry = lookup_one_len(name, parent, strlen(name));
        if (IS_ERR(dentry))
                goto out;

        if (d_really_is_positive(dentry)) {
                error = -EEXIST;
                goto out1;
        }

        inode = new_inode(dir->i_sb);
        if (!inode) {
                error = -ENOMEM;
                goto out1;
        }

        inode->i_ino = get_next_ino();
        inode->i_mode = mode;
        simple_inode_init_ts(inode);
        inode->i_private = data;
        if (S_ISDIR(mode)) {
                inode->i_op = &simple_dir_inode_operations;
                inode->i_fop = &simple_dir_operations;
                inc_nlink(inode);
                inc_nlink(dir);
        } else if (S_ISLNK(mode)) {
                inode->i_op = iops ? iops : &simple_symlink_inode_operations;
                inode->i_link = data;
        } else {
                inode->i_fop = fops;
        }
        d_instantiate(dentry, inode);
        dget(dentry);
        inode_unlock(dir);
        return dentry;

out1:
        dput(dentry);
        dentry = ERR_PTR(error);
out:
        inode_unlock(dir);
        simple_release_fs(&mount, &mount_count);
        return dentry;
}

/**
 * securityfs_create_file - create a file in the securityfs filesystem
 *
 * @name: a pointer to a string containing the name of the file to create.
 * @mode: the permission that the file should have
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          file will be created in the root of the securityfs filesystem.
 * @data: a pointer to something that the caller will want to get to later
 *        on.  The inode.i_private pointer will point to this value on
 *        the open() call.
 * @fops: a pointer to a struct file_operations that should be used for
 *        this file.
 *
 * This function creates a file in securityfs with the given @name.
 *
 * This function returns a pointer to a dentry if it succeeds.  This
 * pointer must be passed to the securityfs_remove() function when the file is
 * to be removed (no automatic cleanup happens if your module is unloaded,
 * you are responsible here).  If an error occurs, the function will return
 * the error value (via ERR_PTR).
 *
 * If securityfs is not enabled in the kernel, the value %-ENODEV is
 * returned.
 */
struct dentry *securityfs_create_file(const char *name, umode_t mode,
                                      struct dentry *parent, void *data,
                                      const struct file_operations *fops)
{
        return securityfs_create_dentry(name, mode, parent, data, fops, NULL);
}
EXPORT_SYMBOL_GPL(securityfs_create_file);

/**
 * securityfs_create_dir - create a directory in the securityfs filesystem
 *
 * @name: a pointer to a string containing the name of the directory to
 *        create.
 * @parent: a pointer to the parent dentry for this file.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          directory will be created in the root of the securityfs filesystem.
 *
 * This function creates a directory in securityfs with the given @name.
 *
 * This function returns a pointer to a dentry if it succeeds.  This
 * pointer must be passed to the securityfs_remove() function when the file is
 * to be removed (no automatic cleanup happens if your module is unloaded,
 * you are responsible here).  If an error occurs, the function will return
 * the error value (via ERR_PTR).
 *
 * If securityfs is not enabled in the kernel, the value %-ENODEV is
 * returned.
 */
struct dentry *securityfs_create_dir(const char *name, struct dentry *parent)
{
        return securityfs_create_file(name, S_IFDIR | 0755, parent, NULL, NULL);
}
EXPORT_SYMBOL_GPL(securityfs_create_dir);

/**
 * securityfs_create_symlink - create a symlink in the securityfs filesystem
 *
 * @name: a pointer to a string containing the name of the symlink to
 *        create.
 * @parent: a pointer to the parent dentry for the symlink.  This should be a
 *          directory dentry if set.  If this parameter is %NULL, then the
 *          directory will be created in the root of the securityfs filesystem.
 * @target: a pointer to a string containing the name of the symlink's target.
 *          If this parameter is %NULL, then the @iops parameter needs to be
 *          setup to handle .readlink and .get_link inode_operations.
 * @iops: a pointer to the struct inode_operations to use for the symlink. If
 *        this parameter is %NULL, then the default simple_symlink_inode
 *        operations will be used.
 *
 * This function creates a symlink in securityfs with the given @name.
 *
 * This function returns a pointer to a dentry if it succeeds.  This
 * pointer must be passed to the securityfs_remove() function when the file is
 * to be removed (no automatic cleanup happens if your module is unloaded,
 * you are responsible here).  If an error occurs, the function will return
 * the error value (via ERR_PTR).
 *
 * If securityfs is not enabled in the kernel, the value %-ENODEV is
 * returned.
 */
struct dentry *securityfs_create_symlink(const char *name,
                                         struct dentry *parent,
                                         const char *target,
                                         const struct inode_operations *iops)
{
        struct dentry *dent;
        char *link = NULL;

        if (target) {
                link = kstrdup(target, GFP_KERNEL);
                if (!link)
                        return ERR_PTR(-ENOMEM);
        }
        dent = securityfs_create_dentry(name, S_IFLNK | 0444, parent,
                                        link, NULL, iops);
        if (IS_ERR(dent))
                kfree(link);

        return dent;
}
EXPORT_SYMBOL_GPL(securityfs_create_symlink);

/**
 * securityfs_remove - removes a file or directory from the securityfs filesystem
 *
 * @dentry: a pointer to a the dentry of the file or directory to be removed.
 *
 * This function removes a file or directory in securityfs that was previously
 * created with a call to another securityfs function (like
 * securityfs_create_file() or variants thereof.)
 *
 * This function is required to be called in order for the file to be
 * removed. No automatic cleanup of files will happen when a module is
 * removed; you are responsible here.
 */
void securityfs_remove(struct dentry *dentry)
{
        struct inode *dir;

        if (IS_ERR_OR_NULL(dentry))
                return;

        dir = d_inode(dentry->d_parent);
        inode_lock(dir);
        if (simple_positive(dentry)) {
                if (d_is_dir(dentry))
                        simple_rmdir(dir, dentry);
                else
                        simple_unlink(dir, dentry);
                dput(dentry);
        }
        inode_unlock(dir);
        simple_release_fs(&mount, &mount_count);
}
EXPORT_SYMBOL_GPL(securityfs_remove);

static void remove_one(struct dentry *victim)
{
        simple_release_fs(&mount, &mount_count);
}

/**
 * securityfs_recursive_remove - recursively removes a file or directory
 *
 * @dentry: a pointer to a the dentry of the file or directory to be removed.
 *
 * This function recursively removes a file or directory in securityfs that was
 * previously created with a call to another securityfs function (like
 * securityfs_create_file() or variants thereof.)
 */
void securityfs_recursive_remove(struct dentry *dentry)
{
        if (IS_ERR_OR_NULL(dentry))
                return;

        simple_pin_fs(&fs_type, &mount, &mount_count);
        simple_recursive_removal(dentry, remove_one);
        simple_release_fs(&mount, &mount_count);
}
EXPORT_SYMBOL_GPL(securityfs_recursive_remove);

#ifdef CONFIG_SECURITY
static struct dentry *lsm_dentry;
static ssize_t lsm_read(struct file *filp, char __user *buf, size_t count,
                        loff_t *ppos)
{
        return simple_read_from_buffer(buf, count, ppos, lsm_names,
                strlen(lsm_names));
}

static const struct file_operations lsm_ops = {
        .read = lsm_read,
        .llseek = generic_file_llseek,
};
#endif

static int __init securityfs_init(void)
{
        int retval;

        retval = sysfs_create_mount_point(kernel_kobj, "security");
        if (retval)
                return retval;

        retval = register_filesystem(&fs_type);
        if (retval) {
                sysfs_remove_mount_point(kernel_kobj, "security");
                return retval;
        }
#ifdef CONFIG_SECURITY
        lsm_dentry = securityfs_create_file("lsm", 0444, NULL, NULL,
                                                &lsm_ops);
#endif
        return 0;
}
core_initcall(securityfs_init);










































































































































































































































































































































































































































































































































































































































































































































































































    1 



    1 



    1 
    1 








































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * corsair-psu.c - Linux driver for Corsair power supplies with HID sensors interface
 * Copyright (C) 2020 Wilken Gottwalt <wilken.gottwalt@posteo.net>
 */

#include <linux/completion.h>
#include <linux/debugfs.h>
#include <linux/errno.h>
#include <linux/hid.h>
#include <linux/hwmon.h>
#include <linux/hwmon-sysfs.h>
#include <linux/jiffies.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/types.h>

/*
 * Corsair protocol for PSUs
 *
 * message size = 64 bytes (request and response, little endian)
 * request:
 *        [length][command][param0][param1][paramX]...
 * reply:
 *        [echo of length][echo of command][data0][data1][dataX]...
 *
 *        - commands are byte sized opcodes
 *        - length is the sum of all bytes of the commands/params
 *        - the micro-controller of most of these PSUs support concatenation in the request and reply,
 *          but it is better to not rely on this (it is also hard to parse)
 *        - the driver uses raw events to be accessible from userspace (though this is not really
 *          supported, it is just there for convenience, may be removed in the future)
 *        - a reply always starts with the length and command in the same order the request used it
 *        - length of the reply data is specific to the command used
 *        - some of the commands work on a rail and can be switched to a specific rail (0 = 12v,
 *          1 = 5v, 2 = 3.3v)
 *        - the format of the init command 0xFE is swapped length/command bytes
 *        - parameter bytes amount and values are specific to the command (rail setting is the only
 *          one for now that uses non-zero values)
 *        - the driver supports debugfs for values not fitting into the hwmon class
 *        - not every device class (HXi or RMi) supports all commands
 *        - if configured wrong the PSU resets or shuts down, often before actually hitting the
 *          reported critical temperature
 *        - new models like HX1500i Series 2023 have changes in the reported vendor and product
 *          strings, both are slightly longer now, report vendor and product in one string and are
 *          the same now
 */

#define DRIVER_NAME                "corsair-psu"

#define REPLY_SIZE                24 /* max length of a reply to a single command */
#define CMD_BUFFER_SIZE                64
#define CMD_TIMEOUT_MS                250
#define SECONDS_PER_HOUR        (60 * 60)
#define SECONDS_PER_DAY                (SECONDS_PER_HOUR * 24)
#define RAIL_COUNT                3 /* 3v3 + 5v + 12v */
#define TEMP_COUNT                2
#define OCP_MULTI_RAIL                0x02

#define PSU_CMD_SELECT_RAIL        0x00 /* expects length 2 */
#define PSU_CMD_FAN_PWM                0x3B /* the rest of the commands expect length 3 */
#define PSU_CMD_RAIL_VOLTS_HCRIT 0x40
#define PSU_CMD_RAIL_VOLTS_LCRIT 0x44
#define PSU_CMD_RAIL_AMPS_HCRIT        0x46
#define PSU_CMD_TEMP_HCRIT        0x4F
#define PSU_CMD_IN_VOLTS        0x88
#define PSU_CMD_IN_AMPS                0x89
#define PSU_CMD_RAIL_VOLTS        0x8B
#define PSU_CMD_RAIL_AMPS        0x8C
#define PSU_CMD_TEMP0                0x8D
#define PSU_CMD_TEMP1                0x8E
#define PSU_CMD_FAN                0x90
#define PSU_CMD_RAIL_WATTS        0x96
#define PSU_CMD_VEND_STR        0x99
#define PSU_CMD_PROD_STR        0x9A
#define PSU_CMD_TOTAL_UPTIME        0xD1
#define PSU_CMD_UPTIME                0xD2
#define PSU_CMD_OCPMODE                0xD8
#define PSU_CMD_TOTAL_WATTS        0xEE
#define PSU_CMD_FAN_PWM_ENABLE        0xF0
#define PSU_CMD_INIT                0xFE

#define L_IN_VOLTS                "v_in"
#define L_OUT_VOLTS_12V                "v_out +12v"
#define L_OUT_VOLTS_5V                "v_out +5v"
#define L_OUT_VOLTS_3_3V        "v_out +3.3v"
#define L_IN_AMPS                "curr in"
#define L_AMPS_12V                "curr +12v"
#define L_AMPS_5V                "curr +5v"
#define L_AMPS_3_3V                "curr +3.3v"
#define L_FAN                        "psu fan"
#define L_TEMP0                        "vrm temp"
#define L_TEMP1                        "case temp"
#define L_WATTS                        "power total"
#define L_WATTS_12V                "power +12v"
#define L_WATTS_5V                "power +5v"
#define L_WATTS_3_3V                "power +3.3v"

static const char *const label_watts[] = {
        L_WATTS,
        L_WATTS_12V,
        L_WATTS_5V,
        L_WATTS_3_3V
};

static const char *const label_volts[] = {
        L_IN_VOLTS,
        L_OUT_VOLTS_12V,
        L_OUT_VOLTS_5V,
        L_OUT_VOLTS_3_3V
};

static const char *const label_amps[] = {
        L_IN_AMPS,
        L_AMPS_12V,
        L_AMPS_5V,
        L_AMPS_3_3V
};

struct corsairpsu_data {
        struct hid_device *hdev;
        struct device *hwmon_dev;
        struct dentry *debugfs;
        struct completion wait_completion;
        struct mutex lock; /* for locking access to cmd_buffer */
        u8 *cmd_buffer;
        char vendor[REPLY_SIZE];
        char product[REPLY_SIZE];
        long temp_crit[TEMP_COUNT];
        long in_crit[RAIL_COUNT];
        long in_lcrit[RAIL_COUNT];
        long curr_crit[RAIL_COUNT];
        u8 temp_crit_support;
        u8 in_crit_support;
        u8 in_lcrit_support;
        u8 curr_crit_support;
        bool in_curr_cmd_support; /* not all commands are supported on every PSU */
};

/* some values are SMBus LINEAR11 data which need a conversion */
static int corsairpsu_linear11_to_int(const u16 val, const int scale)
{
        const int exp = ((s16)val) >> 11;
        const int mant = (((s16)(val & 0x7ff)) << 5) >> 5;
        const int result = mant * scale;

        return (exp >= 0) ? (result << exp) : (result >> -exp);
}

/* the micro-controller uses percentage values to control pwm */
static int corsairpsu_dutycycle_to_pwm(const long dutycycle)
{
        const int result = (256 << 16) / 100;

        return (result * dutycycle) >> 16;
}

static int corsairpsu_usb_cmd(struct corsairpsu_data *priv, u8 p0, u8 p1, u8 p2, void *data)
{
        unsigned long time;
        int ret;

        memset(priv->cmd_buffer, 0, CMD_BUFFER_SIZE);
        priv->cmd_buffer[0] = p0;
        priv->cmd_buffer[1] = p1;
        priv->cmd_buffer[2] = p2;

        reinit_completion(&priv->wait_completion);

        ret = hid_hw_output_report(priv->hdev, priv->cmd_buffer, CMD_BUFFER_SIZE);
        if (ret < 0)
                return ret;

        time = wait_for_completion_timeout(&priv->wait_completion,
                                           msecs_to_jiffies(CMD_TIMEOUT_MS));
        if (!time)
                return -ETIMEDOUT;

        /*
         * at the start of the reply is an echo of the send command/length in the same order it
         * was send, not every command is supported on every device class, if a command is not
         * supported, the length value in the reply is okay, but the command value is set to 0
         */
        if (p0 != priv->cmd_buffer[0] || p1 != priv->cmd_buffer[1])
                return -EOPNOTSUPP;

        if (data)
                memcpy(data, priv->cmd_buffer + 2, REPLY_SIZE);

        return 0;
}

static int corsairpsu_init(struct corsairpsu_data *priv)
{
        /*
         * PSU_CMD_INIT uses swapped length/command and expects 2 parameter bytes, this command
         * actually generates a reply, but we don't need it
         */
        return corsairpsu_usb_cmd(priv, PSU_CMD_INIT, 3, 0, NULL);
}

static int corsairpsu_fwinfo(struct corsairpsu_data *priv)
{
        int ret;

        ret = corsairpsu_usb_cmd(priv, 3, PSU_CMD_VEND_STR, 0, priv->vendor);
        if (ret < 0)
                return ret;

        ret = corsairpsu_usb_cmd(priv, 3, PSU_CMD_PROD_STR, 0, priv->product);
        if (ret < 0)
                return ret;

        return 0;
}

static int corsairpsu_request(struct corsairpsu_data *priv, u8 cmd, u8 rail, void *data)
{
        int ret;

        mutex_lock(&priv->lock);
        switch (cmd) {
        case PSU_CMD_RAIL_VOLTS_HCRIT:
        case PSU_CMD_RAIL_VOLTS_LCRIT:
        case PSU_CMD_RAIL_AMPS_HCRIT:
        case PSU_CMD_RAIL_VOLTS:
        case PSU_CMD_RAIL_AMPS:
        case PSU_CMD_RAIL_WATTS:
                ret = corsairpsu_usb_cmd(priv, 2, PSU_CMD_SELECT_RAIL, rail, NULL);
                if (ret < 0)
                        goto cmd_fail;
                break;
        default:
                break;
        }

        ret = corsairpsu_usb_cmd(priv, 3, cmd, 0, data);

cmd_fail:
        mutex_unlock(&priv->lock);
        return ret;
}

static int corsairpsu_get_value(struct corsairpsu_data *priv, u8 cmd, u8 rail, long *val)
{
        u8 data[REPLY_SIZE];
        long tmp;
        int ret;

        ret = corsairpsu_request(priv, cmd, rail, data);
        if (ret < 0)
                return ret;

        /*
         * the biggest value here comes from the uptime command and to exceed MAXINT total uptime
         * needs to be about 68 years, the rest are u16 values and the biggest value coming out of
         * the LINEAR11 conversion are the watts values which are about 1500 for the strongest psu
         * supported (HX1500i)
         */
        tmp = ((long)data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0];
        switch (cmd) {
        case PSU_CMD_RAIL_VOLTS_HCRIT:
        case PSU_CMD_RAIL_VOLTS_LCRIT:
        case PSU_CMD_RAIL_AMPS_HCRIT:
        case PSU_CMD_TEMP_HCRIT:
        case PSU_CMD_IN_VOLTS:
        case PSU_CMD_IN_AMPS:
        case PSU_CMD_RAIL_VOLTS:
        case PSU_CMD_RAIL_AMPS:
        case PSU_CMD_TEMP0:
        case PSU_CMD_TEMP1:
                *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1000);
                break;
        case PSU_CMD_FAN:
                *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1);
                break;
        case PSU_CMD_FAN_PWM_ENABLE:
                *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1);
                /*
                 * 0 = automatic mode, means the micro-controller controls the fan using a plan
                 *     which can be modified, but changing this plan is not supported by this
                 *     driver, the matching PWM mode is automatic fan speed control = PWM 2
                 * 1 = fixed mode, fan runs at a fixed speed represented by a percentage
                 *     value 0-100, this matches the PWM manual fan speed control = PWM 1
                 * technically there is no PWM no fan speed control mode, it would be a combination
                 * of 1 at 100%
                 */
                if (*val == 0)
                        *val = 2;
                break;
        case PSU_CMD_FAN_PWM:
                *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1);
                *val = corsairpsu_dutycycle_to_pwm(*val);
                break;
        case PSU_CMD_RAIL_WATTS:
        case PSU_CMD_TOTAL_WATTS:
                *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1000000);
                break;
        case PSU_CMD_TOTAL_UPTIME:
        case PSU_CMD_UPTIME:
        case PSU_CMD_OCPMODE:
                *val = tmp;
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }

        return ret;
}

static void corsairpsu_get_criticals(struct corsairpsu_data *priv)
{
        long tmp;
        int rail;

        for (rail = 0; rail < TEMP_COUNT; ++rail) {
                if (!corsairpsu_get_value(priv, PSU_CMD_TEMP_HCRIT, rail, &tmp)) {
                        priv->temp_crit_support |= BIT(rail);
                        priv->temp_crit[rail] = tmp;
                }
        }

        for (rail = 0; rail < RAIL_COUNT; ++rail) {
                if (!corsairpsu_get_value(priv, PSU_CMD_RAIL_VOLTS_HCRIT, rail, &tmp)) {
                        priv->in_crit_support |= BIT(rail);
                        priv->in_crit[rail] = tmp;
                }

                if (!corsairpsu_get_value(priv, PSU_CMD_RAIL_VOLTS_LCRIT, rail, &tmp)) {
                        priv->in_lcrit_support |= BIT(rail);
                        priv->in_lcrit[rail] = tmp;
                }

                if (!corsairpsu_get_value(priv, PSU_CMD_RAIL_AMPS_HCRIT, rail, &tmp)) {
                        priv->curr_crit_support |= BIT(rail);
                        priv->curr_crit[rail] = tmp;
                }
        }
}

static void corsairpsu_check_cmd_support(struct corsairpsu_data *priv)
{
        long tmp;

        priv->in_curr_cmd_support = !corsairpsu_get_value(priv, PSU_CMD_IN_AMPS, 0, &tmp);
}

static umode_t corsairpsu_hwmon_temp_is_visible(const struct corsairpsu_data *priv, u32 attr,
                                                int channel)
{
        umode_t res = 0444;

        switch (attr) {
        case hwmon_temp_input:
        case hwmon_temp_label:
        case hwmon_temp_crit:
                if (channel > 0 && !(priv->temp_crit_support & BIT(channel - 1)))
                        res = 0;
                break;
        default:
                break;
        }

        return res;
}

static umode_t corsairpsu_hwmon_fan_is_visible(const struct corsairpsu_data *priv, u32 attr,
                                               int channel)
{
        switch (attr) {
        case hwmon_fan_input:
        case hwmon_fan_label:
                return 0444;
        default:
                return 0;
        }
}

static umode_t corsairpsu_hwmon_pwm_is_visible(const struct corsairpsu_data *priv, u32 attr,
                                               int channel)
{
        switch (attr) {
        case hwmon_pwm_input:
        case hwmon_pwm_enable:
                return 0444;
        default:
                return 0;
        }
}

static umode_t corsairpsu_hwmon_power_is_visible(const struct corsairpsu_data *priv, u32 attr,
                                                 int channel)
{
        switch (attr) {
        case hwmon_power_input:
        case hwmon_power_label:
                return 0444;
        default:
                return 0;
        }
}

static umode_t corsairpsu_hwmon_in_is_visible(const struct corsairpsu_data *priv, u32 attr,
                                              int channel)
{
        umode_t res = 0444;

        switch (attr) {
        case hwmon_in_input:
        case hwmon_in_label:
        case hwmon_in_crit:
                if (channel > 0 && !(priv->in_crit_support & BIT(channel - 1)))
                        res = 0;
                break;
        case hwmon_in_lcrit:
                if (channel > 0 && !(priv->in_lcrit_support & BIT(channel - 1)))
                        res = 0;
                break;
        default:
                break;
        }

        return res;
}

static umode_t corsairpsu_hwmon_curr_is_visible(const struct corsairpsu_data *priv, u32 attr,
                                                int channel)
{
        umode_t res = 0444;

        switch (attr) {
        case hwmon_curr_input:
                if (channel == 0 && !priv->in_curr_cmd_support)
                        res = 0;
                break;
        case hwmon_curr_label:
        case hwmon_curr_crit:
                if (channel > 0 && !(priv->curr_crit_support & BIT(channel - 1)))
                        res = 0;
                break;
        default:
                break;
        }

        return res;
}

static umode_t corsairpsu_hwmon_ops_is_visible(const void *data, enum hwmon_sensor_types type,
                                               u32 attr, int channel)
{
        const struct corsairpsu_data *priv = data;

        switch (type) {
        case hwmon_temp:
                return corsairpsu_hwmon_temp_is_visible(priv, attr, channel);
        case hwmon_fan:
                return corsairpsu_hwmon_fan_is_visible(priv, attr, channel);
        case hwmon_pwm:
                return corsairpsu_hwmon_pwm_is_visible(priv, attr, channel);
        case hwmon_power:
                return corsairpsu_hwmon_power_is_visible(priv, attr, channel);
        case hwmon_in:
                return corsairpsu_hwmon_in_is_visible(priv, attr, channel);
        case hwmon_curr:
                return corsairpsu_hwmon_curr_is_visible(priv, attr, channel);
        default:
                return 0;
        }
}

static int corsairpsu_hwmon_temp_read(struct corsairpsu_data *priv, u32 attr, int channel,
                                      long *val)
{
        int err = -EOPNOTSUPP;

        switch (attr) {
        case hwmon_temp_input:
                return corsairpsu_get_value(priv, channel ? PSU_CMD_TEMP1 : PSU_CMD_TEMP0,
                                            channel, val);
        case hwmon_temp_crit:
                *val = priv->temp_crit[channel];
                err = 0;
                break;
        default:
                break;
        }

        return err;
}

static int corsairpsu_hwmon_pwm_read(struct corsairpsu_data *priv, u32 attr, int channel, long *val)
{
        switch (attr) {
        case hwmon_pwm_input:
                return corsairpsu_get_value(priv, PSU_CMD_FAN_PWM, 0, val);
        case hwmon_pwm_enable:
                return corsairpsu_get_value(priv, PSU_CMD_FAN_PWM_ENABLE, 0, val);
        default:
                break;
        }

        return -EOPNOTSUPP;
}

static int corsairpsu_hwmon_power_read(struct corsairpsu_data *priv, u32 attr, int channel,
                                       long *val)
{
        if (attr == hwmon_power_input) {
                switch (channel) {
                case 0:
                        return corsairpsu_get_value(priv, PSU_CMD_TOTAL_WATTS, 0, val);
                case 1 ... 3:
                        return corsairpsu_get_value(priv, PSU_CMD_RAIL_WATTS, channel - 1, val);
                default:
                        break;
                }
        }

        return -EOPNOTSUPP;
}

static int corsairpsu_hwmon_in_read(struct corsairpsu_data *priv, u32 attr, int channel, long *val)
{
        int err = -EOPNOTSUPP;

        switch (attr) {
        case hwmon_in_input:
                switch (channel) {
                case 0:
                        return corsairpsu_get_value(priv, PSU_CMD_IN_VOLTS, 0, val);
                case 1 ... 3:
                        return corsairpsu_get_value(priv, PSU_CMD_RAIL_VOLTS, channel - 1, val);
                default:
                        break;
                }
                break;
        case hwmon_in_crit:
                *val = priv->in_crit[channel - 1];
                err = 0;
                break;
        case hwmon_in_lcrit:
                *val = priv->in_lcrit[channel - 1];
                err = 0;
                break;
        }

        return err;
}

static int corsairpsu_hwmon_curr_read(struct corsairpsu_data *priv, u32 attr, int channel,
                                      long *val)
{
        int err = -EOPNOTSUPP;

        switch (attr) {
        case hwmon_curr_input:
                switch (channel) {
                case 0:
                        return corsairpsu_get_value(priv, PSU_CMD_IN_AMPS, 0, val);
                case 1 ... 3:
                        return corsairpsu_get_value(priv, PSU_CMD_RAIL_AMPS, channel - 1, val);
                default:
                        break;
                }
                break;
        case hwmon_curr_crit:
                *val = priv->curr_crit[channel - 1];
                err = 0;
                break;
        default:
                break;
        }

        return err;
}

static int corsairpsu_hwmon_ops_read(struct device *dev, enum hwmon_sensor_types type, u32 attr,
                                     int channel, long *val)
{
        struct corsairpsu_data *priv = dev_get_drvdata(dev);

        switch (type) {
        case hwmon_temp:
                return corsairpsu_hwmon_temp_read(priv, attr, channel, val);
        case hwmon_fan:
                if (attr == hwmon_fan_input)
                        return corsairpsu_get_value(priv, PSU_CMD_FAN, 0, val);
                return -EOPNOTSUPP;
        case hwmon_pwm:
                return corsairpsu_hwmon_pwm_read(priv, attr, channel, val);
        case hwmon_power:
                return corsairpsu_hwmon_power_read(priv, attr, channel, val);
        case hwmon_in:
                return corsairpsu_hwmon_in_read(priv, attr, channel, val);
        case hwmon_curr:
                return corsairpsu_hwmon_curr_read(priv, attr, channel, val);
        default:
                return -EOPNOTSUPP;
        }
}

static int corsairpsu_hwmon_ops_read_string(struct device *dev, enum hwmon_sensor_types type,
                                            u32 attr, int channel, const char **str)
{
        if (type == hwmon_temp && attr == hwmon_temp_label) {
                *str = channel ? L_TEMP1 : L_TEMP0;
                return 0;
        } else if (type == hwmon_fan && attr == hwmon_fan_label) {
                *str = L_FAN;
                return 0;
        } else if (type == hwmon_power && attr == hwmon_power_label && channel < 4) {
                *str = label_watts[channel];
                return 0;
        } else if (type == hwmon_in && attr == hwmon_in_label && channel < 4) {
                *str = label_volts[channel];
                return 0;
        } else if (type == hwmon_curr && attr == hwmon_curr_label && channel < 4) {
                *str = label_amps[channel];
                return 0;
        }

        return -EOPNOTSUPP;
}

static const struct hwmon_ops corsairpsu_hwmon_ops = {
        .is_visible        = corsairpsu_hwmon_ops_is_visible,
        .read                = corsairpsu_hwmon_ops_read,
        .read_string        = corsairpsu_hwmon_ops_read_string,
};

static const struct hwmon_channel_info *const corsairpsu_info[] = {
        HWMON_CHANNEL_INFO(chip,
                           HWMON_C_REGISTER_TZ),
        HWMON_CHANNEL_INFO(temp,
                           HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_CRIT,
                           HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_CRIT),
        HWMON_CHANNEL_INFO(fan,
                           HWMON_F_INPUT | HWMON_F_LABEL),
        HWMON_CHANNEL_INFO(pwm,
                           HWMON_PWM_INPUT | HWMON_PWM_ENABLE),
        HWMON_CHANNEL_INFO(power,
                           HWMON_P_INPUT | HWMON_P_LABEL,
                           HWMON_P_INPUT | HWMON_P_LABEL,
                           HWMON_P_INPUT | HWMON_P_LABEL,
                           HWMON_P_INPUT | HWMON_P_LABEL),
        HWMON_CHANNEL_INFO(in,
                           HWMON_I_INPUT | HWMON_I_LABEL,
                           HWMON_I_INPUT | HWMON_I_LABEL | HWMON_I_LCRIT | HWMON_I_CRIT,
                           HWMON_I_INPUT | HWMON_I_LABEL | HWMON_I_LCRIT | HWMON_I_CRIT,
                           HWMON_I_INPUT | HWMON_I_LABEL | HWMON_I_LCRIT | HWMON_I_CRIT),
        HWMON_CHANNEL_INFO(curr,
                           HWMON_C_INPUT | HWMON_C_LABEL,
                           HWMON_C_INPUT | HWMON_C_LABEL | HWMON_C_CRIT,
                           HWMON_C_INPUT | HWMON_C_LABEL | HWMON_C_CRIT,
                           HWMON_C_INPUT | HWMON_C_LABEL | HWMON_C_CRIT),
        NULL
};

static const struct hwmon_chip_info corsairpsu_chip_info = {
        .ops        = &corsairpsu_hwmon_ops,
        .info        = corsairpsu_info,
};

#ifdef CONFIG_DEBUG_FS

static void print_uptime(struct seq_file *seqf, u8 cmd)
{
        struct corsairpsu_data *priv = seqf->private;
        long val;
        int ret;

        ret = corsairpsu_get_value(priv, cmd, 0, &val);
        if (ret < 0) {
                seq_puts(seqf, "N/A\n");
                return;
        }

        if (val > SECONDS_PER_DAY) {
                seq_printf(seqf, "%ld day(s), %02ld:%02ld:%02ld\n", val / SECONDS_PER_DAY,
                           val % SECONDS_PER_DAY / SECONDS_PER_HOUR, val % SECONDS_PER_HOUR / 60,
                           val % 60);
                return;
        }

        seq_printf(seqf, "%02ld:%02ld:%02ld\n", val % SECONDS_PER_DAY / SECONDS_PER_HOUR,
                   val % SECONDS_PER_HOUR / 60, val % 60);
}

static int uptime_show(struct seq_file *seqf, void *unused)
{
        print_uptime(seqf, PSU_CMD_UPTIME);

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(uptime);

static int uptime_total_show(struct seq_file *seqf, void *unused)
{
        print_uptime(seqf, PSU_CMD_TOTAL_UPTIME);

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(uptime_total);

static int vendor_show(struct seq_file *seqf, void *unused)
{
        struct corsairpsu_data *priv = seqf->private;

        seq_printf(seqf, "%s\n", priv->vendor);

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(vendor);

static int product_show(struct seq_file *seqf, void *unused)
{
        struct corsairpsu_data *priv = seqf->private;

        seq_printf(seqf, "%s\n", priv->product);

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(product);

static int ocpmode_show(struct seq_file *seqf, void *unused)
{
        struct corsairpsu_data *priv = seqf->private;
        long val;
        int ret;

        /*
         * The rail mode is switchable on the fly. The RAW interface can be used for this. But it
         * will not be included here, because I consider it somewhat dangerous for the health of the
         * PSU. The returned value can be a bogus one, if the PSU is in the process of switching and
         * getting of the value itself can also fail during this. Because of this every other value
         * than OCP_MULTI_RAIL can be considered as "single rail".
         */
        ret = corsairpsu_get_value(priv, PSU_CMD_OCPMODE, 0, &val);
        if (ret < 0)
                seq_puts(seqf, "N/A\n");
        else
                seq_printf(seqf, "%s\n", (val == OCP_MULTI_RAIL) ? "multi rail" : "single rail");

        return 0;
}
DEFINE_SHOW_ATTRIBUTE(ocpmode);

static void corsairpsu_debugfs_init(struct corsairpsu_data *priv)
{
        char name[32];

        scnprintf(name, sizeof(name), "%s-%s", DRIVER_NAME, dev_name(&priv->hdev->dev));

        priv->debugfs = debugfs_create_dir(name, NULL);
        debugfs_create_file("uptime", 0444, priv->debugfs, priv, &uptime_fops);
        debugfs_create_file("uptime_total", 0444, priv->debugfs, priv, &uptime_total_fops);
        debugfs_create_file("vendor", 0444, priv->debugfs, priv, &vendor_fops);
        debugfs_create_file("product", 0444, priv->debugfs, priv, &product_fops);
        debugfs_create_file("ocpmode", 0444, priv->debugfs, priv, &ocpmode_fops);
}

#else

static void corsairpsu_debugfs_init(struct corsairpsu_data *priv)
{
}

#endif

static int corsairpsu_probe(struct hid_device *hdev, const struct hid_device_id *id)
{
        struct corsairpsu_data *priv;
        int ret;

        priv = devm_kzalloc(&hdev->dev, sizeof(struct corsairpsu_data), GFP_KERNEL);
        if (!priv)
                return -ENOMEM;

        priv->cmd_buffer = devm_kmalloc(&hdev->dev, CMD_BUFFER_SIZE, GFP_KERNEL);
        if (!priv->cmd_buffer)
                return -ENOMEM;

        ret = hid_parse(hdev);
        if (ret)
                return ret;

        ret = hid_hw_start(hdev, HID_CONNECT_HIDRAW);
        if (ret)
                return ret;

        ret = hid_hw_open(hdev);
        if (ret)
                goto fail_and_stop;

        priv->hdev = hdev;
        hid_set_drvdata(hdev, priv);
        mutex_init(&priv->lock);
        init_completion(&priv->wait_completion);

        hid_device_io_start(hdev);

        ret = corsairpsu_init(priv);
        if (ret < 0) {
                dev_err(&hdev->dev, "unable to initialize device (%d)\n", ret);
                goto fail_and_stop;
        }

        ret = corsairpsu_fwinfo(priv);
        if (ret < 0) {
                dev_err(&hdev->dev, "unable to query firmware (%d)\n", ret);
                goto fail_and_stop;
        }

        corsairpsu_get_criticals(priv);
        corsairpsu_check_cmd_support(priv);

        priv->hwmon_dev = hwmon_device_register_with_info(&hdev->dev, "corsairpsu", priv,
                                                          &corsairpsu_chip_info, NULL);

        if (IS_ERR(priv->hwmon_dev)) {
                ret = PTR_ERR(priv->hwmon_dev);
                goto fail_and_close;
        }

        corsairpsu_debugfs_init(priv);

        return 0;

fail_and_close:
        hid_hw_close(hdev);
fail_and_stop:
        hid_hw_stop(hdev);
        return ret;
}

static void corsairpsu_remove(struct hid_device *hdev)
{
        struct corsairpsu_data *priv = hid_get_drvdata(hdev);

        debugfs_remove_recursive(priv->debugfs);
        hwmon_device_unregister(priv->hwmon_dev);
        hid_hw_close(hdev);
        hid_hw_stop(hdev);
}

static int corsairpsu_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data,
                                int size)
{
        struct corsairpsu_data *priv = hid_get_drvdata(hdev);

        if (completion_done(&priv->wait_completion))
                return 0;

        memcpy(priv->cmd_buffer, data, min(CMD_BUFFER_SIZE, size));
        complete(&priv->wait_completion);

        return 0;
}

#ifdef CONFIG_PM
static int corsairpsu_resume(struct hid_device *hdev)
{
        struct corsairpsu_data *priv = hid_get_drvdata(hdev);

        /* some PSUs turn off the microcontroller during standby, so a reinit is required */
        return corsairpsu_init(priv);
}
#endif

static const struct hid_device_id corsairpsu_idtable[] = {
        { HID_USB_DEVICE(0x1b1c, 0x1c03) }, /* Corsair HX550i */
        { HID_USB_DEVICE(0x1b1c, 0x1c04) }, /* Corsair HX650i */
        { HID_USB_DEVICE(0x1b1c, 0x1c05) }, /* Corsair HX750i */
        { HID_USB_DEVICE(0x1b1c, 0x1c06) }, /* Corsair HX850i */
        { HID_USB_DEVICE(0x1b1c, 0x1c07) }, /* Corsair HX1000i Legacy */
        { HID_USB_DEVICE(0x1b1c, 0x1c08) }, /* Corsair HX1200i Legacy */
        { HID_USB_DEVICE(0x1b1c, 0x1c09) }, /* Corsair RM550i */
        { HID_USB_DEVICE(0x1b1c, 0x1c0a) }, /* Corsair RM650i */
        { HID_USB_DEVICE(0x1b1c, 0x1c0b) }, /* Corsair RM750i */
        { HID_USB_DEVICE(0x1b1c, 0x1c0c) }, /* Corsair RM850i */
        { HID_USB_DEVICE(0x1b1c, 0x1c0d) }, /* Corsair RM1000i */
        { HID_USB_DEVICE(0x1b1c, 0x1c1e) }, /* Corsair HX1000i Series 2023 */
        { HID_USB_DEVICE(0x1b1c, 0x1c1f) }, /* Corsair HX1500i Legacy and Series 2023 */
        { HID_USB_DEVICE(0x1b1c, 0x1c23) }, /* Corsair HX1200i Series 2023 */
        { },
};
MODULE_DEVICE_TABLE(hid, corsairpsu_idtable);

static struct hid_driver corsairpsu_driver = {
        .name                = DRIVER_NAME,
        .id_table        = corsairpsu_idtable,
        .probe                = corsairpsu_probe,
        .remove                = corsairpsu_remove,
        .raw_event        = corsairpsu_raw_event,
#ifdef CONFIG_PM
        .resume                = corsairpsu_resume,
        .reset_resume        = corsairpsu_resume,
#endif
};

static int __init corsair_init(void)
{
        return hid_register_driver(&corsairpsu_driver);
}

static void __exit corsair_exit(void)
{
        hid_unregister_driver(&corsairpsu_driver);
}

/*
 * With module_init() the driver would load before the HID bus when
 * built-in, so use late_initcall() instead.
 */
late_initcall(corsair_init);
module_exit(corsair_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Wilken Gottwalt <wilken.gottwalt@posteo.net>");
MODULE_DESCRIPTION("Linux driver for Corsair power supplies with HID sensors interface");





























    1 




















    1 


    1 

    1 
    1 










    1 

    1 






    1 



















    1 
    1 







    1 








    1 

    1 
    1 





    1 







    2 
    2 


    2 







    2 

    2 
    2 





    2 







    2 
    2 


    2 







    1 

    1 
    1 





    1 







    1 
    1 


    1 






































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
// SPDX-License-Identifier: GPL-2.0-or-later
/* xfrm6_protocol.c - Generic xfrm protocol multiplexer for ipv6.
 *
 * Copyright (C) 2013 secunet Security Networks AG
 *
 * Author:
 * Steffen Klassert <steffen.klassert@secunet.com>
 *
 * Based on:
 * net/ipv4/xfrm4_protocol.c
 */

#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/skbuff.h>
#include <linux/icmpv6.h>
#include <net/ip6_route.h>
#include <net/ipv6.h>
#include <net/protocol.h>
#include <net/xfrm.h>

static struct xfrm6_protocol __rcu *esp6_handlers __read_mostly;
static struct xfrm6_protocol __rcu *ah6_handlers __read_mostly;
static struct xfrm6_protocol __rcu *ipcomp6_handlers __read_mostly;
static DEFINE_MUTEX(xfrm6_protocol_mutex);

static inline struct xfrm6_protocol __rcu **proto_handlers(u8 protocol)
{
        switch (protocol) {
        case IPPROTO_ESP:
                return &esp6_handlers;
        case IPPROTO_AH:
                return &ah6_handlers;
        case IPPROTO_COMP:
                return &ipcomp6_handlers;
        }

        return NULL;
}

#define for_each_protocol_rcu(head, handler)                \
        for (handler = rcu_dereference(head);                \
             handler != NULL;                                \
             handler = rcu_dereference(handler->next))        \

static int xfrm6_rcv_cb(struct sk_buff *skb, u8 protocol, int err)
{
        int ret;
        struct xfrm6_protocol *handler;
        struct xfrm6_protocol __rcu **head = proto_handlers(protocol);

        if (!head)
                return 0;

        for_each_protocol_rcu(*proto_handlers(protocol), handler)
                if ((ret = handler->cb_handler(skb, err)) <= 0)
                        return ret;

        return 0;
}

int xfrm6_rcv_encap(struct sk_buff *skb, int nexthdr, __be32 spi,
                    int encap_type)
{
        int ret;
        struct xfrm6_protocol *handler;
        struct xfrm6_protocol __rcu **head = proto_handlers(nexthdr);

        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
        XFRM_SPI_SKB_CB(skb)->family = AF_INET6;
        XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);

        if (!head)
                goto out;

        if (!skb_dst(skb)) {
                const struct ipv6hdr *ip6h = ipv6_hdr(skb);
                int flags = RT6_LOOKUP_F_HAS_SADDR;
                struct dst_entry *dst;
                struct flowi6 fl6 = {
                        .flowi6_iif   = skb->dev->ifindex,
                        .daddr        = ip6h->daddr,
                        .saddr        = ip6h->saddr,
                        .flowlabel    = ip6_flowinfo(ip6h),
                        .flowi6_mark  = skb->mark,
                        .flowi6_proto = ip6h->nexthdr,
                };

                dst = ip6_route_input_lookup(dev_net(skb->dev), skb->dev, &fl6,
                                             skb, flags);
                if (dst->error)
                        goto drop;
                skb_dst_set(skb, dst);
        }

        for_each_protocol_rcu(*head, handler)
                if ((ret = handler->input_handler(skb, nexthdr, spi, encap_type)) != -EINVAL)
                        return ret;

out:
        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

drop:
        kfree_skb(skb);
        return 0;
}
EXPORT_SYMBOL(xfrm6_rcv_encap);

static int xfrm6_esp_rcv(struct sk_buff *skb)
{
        int ret;
        struct xfrm6_protocol *handler;

        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;

        for_each_protocol_rcu(esp6_handlers, handler)
                if ((ret = handler->handler(skb)) != -EINVAL)
                        return ret;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

        kfree_skb(skb);
        return 0;
}

static int xfrm6_esp_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                          u8 type, u8 code, int offset, __be32 info)
{
        struct xfrm6_protocol *handler;

        for_each_protocol_rcu(esp6_handlers, handler)
                if (!handler->err_handler(skb, opt, type, code, offset, info))
                        return 0;

        return -ENOENT;
}

static int xfrm6_ah_rcv(struct sk_buff *skb)
{
        int ret;
        struct xfrm6_protocol *handler;

        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;

        for_each_protocol_rcu(ah6_handlers, handler)
                if ((ret = handler->handler(skb)) != -EINVAL)
                        return ret;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

        kfree_skb(skb);
        return 0;
}

static int xfrm6_ah_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                         u8 type, u8 code, int offset, __be32 info)
{
        struct xfrm6_protocol *handler;

        for_each_protocol_rcu(ah6_handlers, handler)
                if (!handler->err_handler(skb, opt, type, code, offset, info))
                        return 0;

        return -ENOENT;
}

static int xfrm6_ipcomp_rcv(struct sk_buff *skb)
{
        int ret;
        struct xfrm6_protocol *handler;

        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;

        for_each_protocol_rcu(ipcomp6_handlers, handler)
                if ((ret = handler->handler(skb)) != -EINVAL)
                        return ret;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

        kfree_skb(skb);
        return 0;
}

static int xfrm6_ipcomp_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                             u8 type, u8 code, int offset, __be32 info)
{
        struct xfrm6_protocol *handler;

        for_each_protocol_rcu(ipcomp6_handlers, handler)
                if (!handler->err_handler(skb, opt, type, code, offset, info))
                        return 0;

        return -ENOENT;
}

static const struct inet6_protocol esp6_protocol = {
        .handler        =        xfrm6_esp_rcv,
        .err_handler        =        xfrm6_esp_err,
        .flags                =        INET6_PROTO_NOPOLICY,
};

static const struct inet6_protocol ah6_protocol = {
        .handler        =        xfrm6_ah_rcv,
        .err_handler        =        xfrm6_ah_err,
        .flags                =        INET6_PROTO_NOPOLICY,
};

static const struct inet6_protocol ipcomp6_protocol = {
        .handler        =        xfrm6_ipcomp_rcv,
        .err_handler        =        xfrm6_ipcomp_err,
        .flags                =        INET6_PROTO_NOPOLICY,
};

static const struct xfrm_input_afinfo xfrm6_input_afinfo = {
        .family                =        AF_INET6,
        .callback        =        xfrm6_rcv_cb,
};

static inline const struct inet6_protocol *netproto(unsigned char protocol)
{
        switch (protocol) {
        case IPPROTO_ESP:
                return &esp6_protocol;
        case IPPROTO_AH:
                return &ah6_protocol;
        case IPPROTO_COMP:
                return &ipcomp6_protocol;
        }

        return NULL;
}

int xfrm6_protocol_register(struct xfrm6_protocol *handler,
                            unsigned char protocol)
{
        struct xfrm6_protocol __rcu **pprev;
        struct xfrm6_protocol *t;
        bool add_netproto = false;
        int ret = -EEXIST;
        int priority = handler->priority;

        if (!proto_handlers(protocol) || !netproto(protocol))
                return -EINVAL;

        mutex_lock(&xfrm6_protocol_mutex);

        if (!rcu_dereference_protected(*proto_handlers(protocol),
                                       lockdep_is_held(&xfrm6_protocol_mutex)))
                add_netproto = true;

        for (pprev = proto_handlers(protocol);
             (t = rcu_dereference_protected(*pprev,
                        lockdep_is_held(&xfrm6_protocol_mutex))) != NULL;
             pprev = &t->next) {
                if (t->priority < priority)
                        break;
                if (t->priority == priority)
                        goto err;
        }

        handler->next = *pprev;
        rcu_assign_pointer(*pprev, handler);

        ret = 0;

err:
        mutex_unlock(&xfrm6_protocol_mutex);

        if (add_netproto) {
                if (inet6_add_protocol(netproto(protocol), protocol)) {
                        pr_err("%s: can't add protocol\n", __func__);
                        ret = -EAGAIN;
                }
        }

        return ret;
}
EXPORT_SYMBOL(xfrm6_protocol_register);

int xfrm6_protocol_deregister(struct xfrm6_protocol *handler,
                              unsigned char protocol)
{
        struct xfrm6_protocol __rcu **pprev;
        struct xfrm6_protocol *t;
        int ret = -ENOENT;

        if (!proto_handlers(protocol) || !netproto(protocol))
                return -EINVAL;

        mutex_lock(&xfrm6_protocol_mutex);

        for (pprev = proto_handlers(protocol);
             (t = rcu_dereference_protected(*pprev,
                        lockdep_is_held(&xfrm6_protocol_mutex))) != NULL;
             pprev = &t->next) {
                if (t == handler) {
                        *pprev = handler->next;
                        ret = 0;
                        break;
                }
        }

        if (!rcu_dereference_protected(*proto_handlers(protocol),
                                       lockdep_is_held(&xfrm6_protocol_mutex))) {
                if (inet6_del_protocol(netproto(protocol), protocol) < 0) {
                        pr_err("%s: can't remove protocol\n", __func__);
                        ret = -EAGAIN;
                }
        }

        mutex_unlock(&xfrm6_protocol_mutex);

        synchronize_net();

        return ret;
}
EXPORT_SYMBOL(xfrm6_protocol_deregister);

int __init xfrm6_protocol_init(void)
{
        return xfrm_input_register_afinfo(&xfrm6_input_afinfo);
}

void xfrm6_protocol_fini(void)
{
        xfrm_input_unregister_afinfo(&xfrm6_input_afinfo);
}

























































    1 















    1 








































    9 

































































































































































































































































































































































































































































    9 


    9 




























    1 














    1 










    1 




















    1 








    1 

    1 









    1 










































































    1 

    1 









    1 



























































































































































































































































































































































































































































































































    9 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
// SPDX-License-Identifier: GPL-2.0-or-later

#include <linux/mrp_bridge.h>
#include "br_private_mrp.h"

static const u8 mrp_test_dmac[ETH_ALEN] = { 0x1, 0x15, 0x4e, 0x0, 0x0, 0x1 };
static const u8 mrp_in_test_dmac[ETH_ALEN] = { 0x1, 0x15, 0x4e, 0x0, 0x0, 0x3 };

static int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb);

static struct br_frame_type mrp_frame_type __read_mostly = {
        .type = cpu_to_be16(ETH_P_MRP),
        .frame_handler = br_mrp_process,
};

static bool br_mrp_is_ring_port(struct net_bridge_port *p_port,
                                struct net_bridge_port *s_port,
                                struct net_bridge_port *port)
{
        if (port == p_port ||
            port == s_port)
                return true;

        return false;
}

static bool br_mrp_is_in_port(struct net_bridge_port *i_port,
                              struct net_bridge_port *port)
{
        if (port == i_port)
                return true;

        return false;
}

static struct net_bridge_port *br_mrp_get_port(struct net_bridge *br,
                                               u32 ifindex)
{
        struct net_bridge_port *res = NULL;
        struct net_bridge_port *port;

        list_for_each_entry(port, &br->port_list, list) {
                if (port->dev->ifindex == ifindex) {
                        res = port;
                        break;
                }
        }

        return res;
}

static struct br_mrp *br_mrp_find_id(struct net_bridge *br, u32 ring_id)
{
        struct br_mrp *res = NULL;
        struct br_mrp *mrp;

        hlist_for_each_entry_rcu(mrp, &br->mrp_list, list,
                                 lockdep_rtnl_is_held()) {
                if (mrp->ring_id == ring_id) {
                        res = mrp;
                        break;
                }
        }

        return res;
}

static struct br_mrp *br_mrp_find_in_id(struct net_bridge *br, u32 in_id)
{
        struct br_mrp *res = NULL;
        struct br_mrp *mrp;

        hlist_for_each_entry_rcu(mrp, &br->mrp_list, list,
                                 lockdep_rtnl_is_held()) {
                if (mrp->in_id == in_id) {
                        res = mrp;
                        break;
                }
        }

        return res;
}

static bool br_mrp_unique_ifindex(struct net_bridge *br, u32 ifindex)
{
        struct br_mrp *mrp;

        hlist_for_each_entry_rcu(mrp, &br->mrp_list, list,
                                 lockdep_rtnl_is_held()) {
                struct net_bridge_port *p;

                p = rtnl_dereference(mrp->p_port);
                if (p && p->dev->ifindex == ifindex)
                        return false;

                p = rtnl_dereference(mrp->s_port);
                if (p && p->dev->ifindex == ifindex)
                        return false;

                p = rtnl_dereference(mrp->i_port);
                if (p && p->dev->ifindex == ifindex)
                        return false;
        }

        return true;
}

static struct br_mrp *br_mrp_find_port(struct net_bridge *br,
                                       struct net_bridge_port *p)
{
        struct br_mrp *res = NULL;
        struct br_mrp *mrp;

        hlist_for_each_entry_rcu(mrp, &br->mrp_list, list,
                                 lockdep_rtnl_is_held()) {
                if (rcu_access_pointer(mrp->p_port) == p ||
                    rcu_access_pointer(mrp->s_port) == p ||
                    rcu_access_pointer(mrp->i_port) == p) {
                        res = mrp;
                        break;
                }
        }

        return res;
}

static int br_mrp_next_seq(struct br_mrp *mrp)
{
        mrp->seq_id++;
        return mrp->seq_id;
}

static struct sk_buff *br_mrp_skb_alloc(struct net_bridge_port *p,
                                        const u8 *src, const u8 *dst)
{
        struct ethhdr *eth_hdr;
        struct sk_buff *skb;
        __be16 *version;

        skb = dev_alloc_skb(MRP_MAX_FRAME_LENGTH);
        if (!skb)
                return NULL;

        skb->dev = p->dev;
        skb->protocol = htons(ETH_P_MRP);
        skb->priority = MRP_FRAME_PRIO;
        skb_reserve(skb, sizeof(*eth_hdr));

        eth_hdr = skb_push(skb, sizeof(*eth_hdr));
        ether_addr_copy(eth_hdr->h_dest, dst);
        ether_addr_copy(eth_hdr->h_source, src);
        eth_hdr->h_proto = htons(ETH_P_MRP);

        version = skb_put(skb, sizeof(*version));
        *version = cpu_to_be16(MRP_VERSION);

        return skb;
}

static void br_mrp_skb_tlv(struct sk_buff *skb,
                           enum br_mrp_tlv_header_type type,
                           u8 length)
{
        struct br_mrp_tlv_hdr *hdr;

        hdr = skb_put(skb, sizeof(*hdr));
        hdr->type = type;
        hdr->length = length;
}

static void br_mrp_skb_common(struct sk_buff *skb, struct br_mrp *mrp)
{
        struct br_mrp_common_hdr *hdr;

        br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_COMMON, sizeof(*hdr));

        hdr = skb_put(skb, sizeof(*hdr));
        hdr->seq_id = cpu_to_be16(br_mrp_next_seq(mrp));
        memset(hdr->domain, 0xff, MRP_DOMAIN_UUID_LENGTH);
}

static struct sk_buff *br_mrp_alloc_test_skb(struct br_mrp *mrp,
                                             struct net_bridge_port *p,
                                             enum br_mrp_port_role_type port_role)
{
        struct br_mrp_ring_test_hdr *hdr = NULL;
        struct sk_buff *skb = NULL;

        if (!p)
                return NULL;

        skb = br_mrp_skb_alloc(p, p->dev->dev_addr, mrp_test_dmac);
        if (!skb)
                return NULL;

        br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_RING_TEST, sizeof(*hdr));
        hdr = skb_put(skb, sizeof(*hdr));

        hdr->prio = cpu_to_be16(mrp->prio);
        ether_addr_copy(hdr->sa, p->br->dev->dev_addr);
        hdr->port_role = cpu_to_be16(port_role);
        hdr->state = cpu_to_be16(mrp->ring_state);
        hdr->transitions = cpu_to_be16(mrp->ring_transitions);
        hdr->timestamp = cpu_to_be32(jiffies_to_msecs(jiffies));

        br_mrp_skb_common(skb, mrp);

        /* In case the node behaves as MRA then the Test frame needs to have
         * an Option TLV which includes eventually a sub-option TLV that has
         * the type AUTO_MGR
         */
        if (mrp->ring_role == BR_MRP_RING_ROLE_MRA) {
                struct br_mrp_sub_option1_hdr *sub_opt = NULL;
                struct br_mrp_tlv_hdr *sub_tlv = NULL;
                struct br_mrp_oui_hdr *oui = NULL;
                u8 length;

                length = sizeof(*sub_opt) + sizeof(*sub_tlv) + sizeof(oui) +
                        MRP_OPT_PADDING;
                br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_OPTION, length);

                oui = skb_put(skb, sizeof(*oui));
                memset(oui, 0x0, sizeof(*oui));
                sub_opt = skb_put(skb, sizeof(*sub_opt));
                memset(sub_opt, 0x0, sizeof(*sub_opt));

                sub_tlv = skb_put(skb, sizeof(*sub_tlv));
                sub_tlv->type = BR_MRP_SUB_TLV_HEADER_TEST_AUTO_MGR;

                /* 32 bit alligment shall be ensured therefore add 2 bytes */
                skb_put(skb, MRP_OPT_PADDING);
        }

        br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_END, 0x0);

        return skb;
}

static struct sk_buff *br_mrp_alloc_in_test_skb(struct br_mrp *mrp,
                                                struct net_bridge_port *p,
                                                enum br_mrp_port_role_type port_role)
{
        struct br_mrp_in_test_hdr *hdr = NULL;
        struct sk_buff *skb = NULL;

        if (!p)
                return NULL;

        skb = br_mrp_skb_alloc(p, p->dev->dev_addr, mrp_in_test_dmac);
        if (!skb)
                return NULL;

        br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_IN_TEST, sizeof(*hdr));
        hdr = skb_put(skb, sizeof(*hdr));

        hdr->id = cpu_to_be16(mrp->in_id);
        ether_addr_copy(hdr->sa, p->br->dev->dev_addr);
        hdr->port_role = cpu_to_be16(port_role);
        hdr->state = cpu_to_be16(mrp->in_state);
        hdr->transitions = cpu_to_be16(mrp->in_transitions);
        hdr->timestamp = cpu_to_be32(jiffies_to_msecs(jiffies));

        br_mrp_skb_common(skb, mrp);
        br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_END, 0x0);

        return skb;
}

/* This function is continuously called in the following cases:
 * - when node role is MRM, in this case test_monitor is always set to false
 *   because it needs to notify the userspace that the ring is open and needs to
 *   send MRP_Test frames
 * - when node role is MRA, there are 2 subcases:
 *     - when MRA behaves as MRM, in this case is similar with MRM role
 *     - when MRA behaves as MRC, in this case test_monitor is set to true,
 *       because it needs to detect when it stops seeing MRP_Test frames
 *       from MRM node but it doesn't need to send MRP_Test frames.
 */
static void br_mrp_test_work_expired(struct work_struct *work)
{
        struct delayed_work *del_work = to_delayed_work(work);
        struct br_mrp *mrp = container_of(del_work, struct br_mrp, test_work);
        struct net_bridge_port *p;
        bool notify_open = false;
        struct sk_buff *skb;

        if (time_before_eq(mrp->test_end, jiffies))
                return;

        if (mrp->test_count_miss < mrp->test_max_miss) {
                mrp->test_count_miss++;
        } else {
                /* Notify that the ring is open only if the ring state is
                 * closed, otherwise it would continue to notify at every
                 * interval.
                 * Also notify that the ring is open when the node has the
                 * role MRA and behaves as MRC. The reason is that the
                 * userspace needs to know when the MRM stopped sending
                 * MRP_Test frames so that the current node to try to take
                 * the role of a MRM.
                 */
                if (mrp->ring_state == BR_MRP_RING_STATE_CLOSED ||
                    mrp->test_monitor)
                        notify_open = true;
        }

        rcu_read_lock();

        p = rcu_dereference(mrp->p_port);
        if (p) {
                if (!mrp->test_monitor) {
                        skb = br_mrp_alloc_test_skb(mrp, p,
                                                    BR_MRP_PORT_ROLE_PRIMARY);
                        if (!skb)
                                goto out;

                        skb_reset_network_header(skb);
                        dev_queue_xmit(skb);
                }

                if (notify_open && !mrp->ring_role_offloaded)
                        br_mrp_ring_port_open(p->dev, true);
        }

        p = rcu_dereference(mrp->s_port);
        if (p) {
                if (!mrp->test_monitor) {
                        skb = br_mrp_alloc_test_skb(mrp, p,
                                                    BR_MRP_PORT_ROLE_SECONDARY);
                        if (!skb)
                                goto out;

                        skb_reset_network_header(skb);
                        dev_queue_xmit(skb);
                }

                if (notify_open && !mrp->ring_role_offloaded)
                        br_mrp_ring_port_open(p->dev, true);
        }

out:
        rcu_read_unlock();

        queue_delayed_work(system_wq, &mrp->test_work,
                           usecs_to_jiffies(mrp->test_interval));
}

/* This function is continuously called when the node has the interconnect role
 * MIM. It would generate interconnect test frames and will send them on all 3
 * ports. But will also check if it stop receiving interconnect test frames.
 */
static void br_mrp_in_test_work_expired(struct work_struct *work)
{
        struct delayed_work *del_work = to_delayed_work(work);
        struct br_mrp *mrp = container_of(del_work, struct br_mrp, in_test_work);
        struct net_bridge_port *p;
        bool notify_open = false;
        struct sk_buff *skb;

        if (time_before_eq(mrp->in_test_end, jiffies))
                return;

        if (mrp->in_test_count_miss < mrp->in_test_max_miss) {
                mrp->in_test_count_miss++;
        } else {
                /* Notify that the interconnect ring is open only if the
                 * interconnect ring state is closed, otherwise it would
                 * continue to notify at every interval.
                 */
                if (mrp->in_state == BR_MRP_IN_STATE_CLOSED)
                        notify_open = true;
        }

        rcu_read_lock();

        p = rcu_dereference(mrp->p_port);
        if (p) {
                skb = br_mrp_alloc_in_test_skb(mrp, p,
                                               BR_MRP_PORT_ROLE_PRIMARY);
                if (!skb)
                        goto out;

                skb_reset_network_header(skb);
                dev_queue_xmit(skb);

                if (notify_open && !mrp->in_role_offloaded)
                        br_mrp_in_port_open(p->dev, true);
        }

        p = rcu_dereference(mrp->s_port);
        if (p) {
                skb = br_mrp_alloc_in_test_skb(mrp, p,
                                               BR_MRP_PORT_ROLE_SECONDARY);
                if (!skb)
                        goto out;

                skb_reset_network_header(skb);
                dev_queue_xmit(skb);

                if (notify_open && !mrp->in_role_offloaded)
                        br_mrp_in_port_open(p->dev, true);
        }

        p = rcu_dereference(mrp->i_port);
        if (p) {
                skb = br_mrp_alloc_in_test_skb(mrp, p,
                                               BR_MRP_PORT_ROLE_INTER);
                if (!skb)
                        goto out;

                skb_reset_network_header(skb);
                dev_queue_xmit(skb);

                if (notify_open && !mrp->in_role_offloaded)
                        br_mrp_in_port_open(p->dev, true);
        }

out:
        rcu_read_unlock();

        queue_delayed_work(system_wq, &mrp->in_test_work,
                           usecs_to_jiffies(mrp->in_test_interval));
}

/* Deletes the MRP instance.
 * note: called under rtnl_lock
 */
static void br_mrp_del_impl(struct net_bridge *br, struct br_mrp *mrp)
{
        struct net_bridge_port *p;
        u8 state;

        /* Stop sending MRP_Test frames */
        cancel_delayed_work_sync(&mrp->test_work);
        br_mrp_switchdev_send_ring_test(br, mrp, 0, 0, 0, 0);

        /* Stop sending MRP_InTest frames if has an interconnect role */
        cancel_delayed_work_sync(&mrp->in_test_work);
        br_mrp_switchdev_send_in_test(br, mrp, 0, 0, 0);

        /* Disable the roles */
        br_mrp_switchdev_set_ring_role(br, mrp, BR_MRP_RING_ROLE_DISABLED);
        p = rtnl_dereference(mrp->i_port);
        if (p)
                br_mrp_switchdev_set_in_role(br, mrp, mrp->in_id, mrp->ring_id,
                                             BR_MRP_IN_ROLE_DISABLED);

        br_mrp_switchdev_del(br, mrp);

        /* Reset the ports */
        p = rtnl_dereference(mrp->p_port);
        if (p) {
                spin_lock_bh(&br->lock);
                state = netif_running(br->dev) ?
                                BR_STATE_FORWARDING : BR_STATE_DISABLED;
                p->state = state;
                p->flags &= ~BR_MRP_AWARE;
                spin_unlock_bh(&br->lock);
                br_mrp_port_switchdev_set_state(p, state);
                rcu_assign_pointer(mrp->p_port, NULL);
        }

        p = rtnl_dereference(mrp->s_port);
        if (p) {
                spin_lock_bh(&br->lock);
                state = netif_running(br->dev) ?
                                BR_STATE_FORWARDING : BR_STATE_DISABLED;
                p->state = state;
                p->flags &= ~BR_MRP_AWARE;
                spin_unlock_bh(&br->lock);
                br_mrp_port_switchdev_set_state(p, state);
                rcu_assign_pointer(mrp->s_port, NULL);
        }

        p = rtnl_dereference(mrp->i_port);
        if (p) {
                spin_lock_bh(&br->lock);
                state = netif_running(br->dev) ?
                                BR_STATE_FORWARDING : BR_STATE_DISABLED;
                p->state = state;
                p->flags &= ~BR_MRP_AWARE;
                spin_unlock_bh(&br->lock);
                br_mrp_port_switchdev_set_state(p, state);
                rcu_assign_pointer(mrp->i_port, NULL);
        }

        hlist_del_rcu(&mrp->list);
        kfree_rcu(mrp, rcu);

        if (hlist_empty(&br->mrp_list))
                br_del_frame(br, &mrp_frame_type);
}

/* Adds a new MRP instance.
 * note: called under rtnl_lock
 */
int br_mrp_add(struct net_bridge *br, struct br_mrp_instance *instance)
{
        struct net_bridge_port *p;
        struct br_mrp *mrp;
        int err;

        /* If the ring exists, it is not possible to create another one with the
         * same ring_id
         */
        mrp = br_mrp_find_id(br, instance->ring_id);
        if (mrp)
                return -EINVAL;

        if (!br_mrp_get_port(br, instance->p_ifindex) ||
            !br_mrp_get_port(br, instance->s_ifindex))
                return -EINVAL;

        /* It is not possible to have the same port part of multiple rings */
        if (!br_mrp_unique_ifindex(br, instance->p_ifindex) ||
            !br_mrp_unique_ifindex(br, instance->s_ifindex))
                return -EINVAL;

        mrp = kzalloc(sizeof(*mrp), GFP_KERNEL);
        if (!mrp)
                return -ENOMEM;

        mrp->ring_id = instance->ring_id;
        mrp->prio = instance->prio;

        p = br_mrp_get_port(br, instance->p_ifindex);
        spin_lock_bh(&br->lock);
        p->state = BR_STATE_FORWARDING;
        p->flags |= BR_MRP_AWARE;
        spin_unlock_bh(&br->lock);
        rcu_assign_pointer(mrp->p_port, p);

        p = br_mrp_get_port(br, instance->s_ifindex);
        spin_lock_bh(&br->lock);
        p->state = BR_STATE_FORWARDING;
        p->flags |= BR_MRP_AWARE;
        spin_unlock_bh(&br->lock);
        rcu_assign_pointer(mrp->s_port, p);

        if (hlist_empty(&br->mrp_list))
                br_add_frame(br, &mrp_frame_type);

        INIT_DELAYED_WORK(&mrp->test_work, br_mrp_test_work_expired);
        INIT_DELAYED_WORK(&mrp->in_test_work, br_mrp_in_test_work_expired);
        hlist_add_tail_rcu(&mrp->list, &br->mrp_list);

        err = br_mrp_switchdev_add(br, mrp);
        if (err)
                goto delete_mrp;

        return 0;

delete_mrp:
        br_mrp_del_impl(br, mrp);

        return err;
}

/* Deletes the MRP instance from which the port is part of
 * note: called under rtnl_lock
 */
void br_mrp_port_del(struct net_bridge *br, struct net_bridge_port *p)
{
        struct br_mrp *mrp = br_mrp_find_port(br, p);

        /* If the port is not part of a MRP instance just bail out */
        if (!mrp)
                return;

        br_mrp_del_impl(br, mrp);
}

/* Deletes existing MRP instance based on ring_id
 * note: called under rtnl_lock
 */
int br_mrp_del(struct net_bridge *br, struct br_mrp_instance *instance)
{
        struct br_mrp *mrp = br_mrp_find_id(br, instance->ring_id);

        if (!mrp)
                return -EINVAL;

        br_mrp_del_impl(br, mrp);

        return 0;
}

/* Set port state, port state can be forwarding, blocked or disabled
 * note: already called with rtnl_lock
 */
int br_mrp_set_port_state(struct net_bridge_port *p,
                          enum br_mrp_port_state_type state)
{
        u32 port_state;

        if (!p || !(p->flags & BR_MRP_AWARE))
                return -EINVAL;

        spin_lock_bh(&p->br->lock);

        if (state == BR_MRP_PORT_STATE_FORWARDING)
                port_state = BR_STATE_FORWARDING;
        else
                port_state = BR_STATE_BLOCKING;

        p->state = port_state;
        spin_unlock_bh(&p->br->lock);

        br_mrp_port_switchdev_set_state(p, port_state);

        return 0;
}

/* Set port role, port role can be primary or secondary
 * note: already called with rtnl_lock
 */
int br_mrp_set_port_role(struct net_bridge_port *p,
                         enum br_mrp_port_role_type role)
{
        struct br_mrp *mrp;

        if (!p || !(p->flags & BR_MRP_AWARE))
                return -EINVAL;

        mrp = br_mrp_find_port(p->br, p);

        if (!mrp)
                return -EINVAL;

        switch (role) {
        case BR_MRP_PORT_ROLE_PRIMARY:
                rcu_assign_pointer(mrp->p_port, p);
                break;
        case BR_MRP_PORT_ROLE_SECONDARY:
                rcu_assign_pointer(mrp->s_port, p);
                break;
        default:
                return -EINVAL;
        }

        br_mrp_port_switchdev_set_role(p, role);

        return 0;
}

/* Set ring state, ring state can be only Open or Closed
 * note: already called with rtnl_lock
 */
int br_mrp_set_ring_state(struct net_bridge *br,
                          struct br_mrp_ring_state *state)
{
        struct br_mrp *mrp = br_mrp_find_id(br, state->ring_id);

        if (!mrp)
                return -EINVAL;

        if (mrp->ring_state != state->ring_state)
                mrp->ring_transitions++;

        mrp->ring_state = state->ring_state;

        br_mrp_switchdev_set_ring_state(br, mrp, state->ring_state);

        return 0;
}

/* Set ring role, ring role can be only MRM(Media Redundancy Manager) or
 * MRC(Media Redundancy Client).
 * note: already called with rtnl_lock
 */
int br_mrp_set_ring_role(struct net_bridge *br,
                         struct br_mrp_ring_role *role)
{
        struct br_mrp *mrp = br_mrp_find_id(br, role->ring_id);
        enum br_mrp_hw_support support;

        if (!mrp)
                return -EINVAL;

        mrp->ring_role = role->ring_role;

        /* If there is an error just bailed out */
        support = br_mrp_switchdev_set_ring_role(br, mrp, role->ring_role);
        if (support == BR_MRP_NONE)
                return -EOPNOTSUPP;

        /* Now detect if the HW actually applied the role or not. If the HW
         * applied the role it means that the SW will not to do those operations
         * anymore. For example if the role ir MRM then the HW will notify the
         * SW when ring is open, but if the is not pushed to the HW the SW will
         * need to detect when the ring is open
         */
        mrp->ring_role_offloaded = support == BR_MRP_SW ? 0 : 1;

        return 0;
}

/* Start to generate or monitor MRP test frames, the frames are generated by
 * HW and if it fails, they are generated by the SW.
 * note: already called with rtnl_lock
 */
int br_mrp_start_test(struct net_bridge *br,
                      struct br_mrp_start_test *test)
{
        struct br_mrp *mrp = br_mrp_find_id(br, test->ring_id);
        enum br_mrp_hw_support support;

        if (!mrp)
                return -EINVAL;

        /* Try to push it to the HW and if it fails then continue with SW
         * implementation and if that also fails then return error.
         */
        support = br_mrp_switchdev_send_ring_test(br, mrp, test->interval,
                                                  test->max_miss, test->period,
                                                  test->monitor);
        if (support == BR_MRP_NONE)
                return -EOPNOTSUPP;

        if (support == BR_MRP_HW)
                return 0;

        mrp->test_interval = test->interval;
        mrp->test_end = jiffies + usecs_to_jiffies(test->period);
        mrp->test_max_miss = test->max_miss;
        mrp->test_monitor = test->monitor;
        mrp->test_count_miss = 0;
        queue_delayed_work(system_wq, &mrp->test_work,
                           usecs_to_jiffies(test->interval));

        return 0;
}

/* Set in state, int state can be only Open or Closed
 * note: already called with rtnl_lock
 */
int br_mrp_set_in_state(struct net_bridge *br, struct br_mrp_in_state *state)
{
        struct br_mrp *mrp = br_mrp_find_in_id(br, state->in_id);

        if (!mrp)
                return -EINVAL;

        if (mrp->in_state != state->in_state)
                mrp->in_transitions++;

        mrp->in_state = state->in_state;

        br_mrp_switchdev_set_in_state(br, mrp, state->in_state);

        return 0;
}

/* Set in role, in role can be only MIM(Media Interconnection Manager) or
 * MIC(Media Interconnection Client).
 * note: already called with rtnl_lock
 */
int br_mrp_set_in_role(struct net_bridge *br, struct br_mrp_in_role *role)
{
        struct br_mrp *mrp = br_mrp_find_id(br, role->ring_id);
        enum br_mrp_hw_support support;
        struct net_bridge_port *p;

        if (!mrp)
                return -EINVAL;

        if (!br_mrp_get_port(br, role->i_ifindex))
                return -EINVAL;

        if (role->in_role == BR_MRP_IN_ROLE_DISABLED) {
                u8 state;

                /* It is not allowed to disable a port that doesn't exist */
                p = rtnl_dereference(mrp->i_port);
                if (!p)
                        return -EINVAL;

                /* Stop the generating MRP_InTest frames */
                cancel_delayed_work_sync(&mrp->in_test_work);
                br_mrp_switchdev_send_in_test(br, mrp, 0, 0, 0);

                /* Remove the port */
                spin_lock_bh(&br->lock);
                state = netif_running(br->dev) ?
                                BR_STATE_FORWARDING : BR_STATE_DISABLED;
                p->state = state;
                p->flags &= ~BR_MRP_AWARE;
                spin_unlock_bh(&br->lock);
                br_mrp_port_switchdev_set_state(p, state);
                rcu_assign_pointer(mrp->i_port, NULL);

                mrp->in_role = role->in_role;
                mrp->in_id = 0;

                return 0;
        }

        /* It is not possible to have the same port part of multiple rings */
        if (!br_mrp_unique_ifindex(br, role->i_ifindex))
                return -EINVAL;

        /* It is not allowed to set a different interconnect port if the mrp
         * instance has already one. First it needs to be disabled and after
         * that set the new port
         */
        if (rcu_access_pointer(mrp->i_port))
                return -EINVAL;

        p = br_mrp_get_port(br, role->i_ifindex);
        spin_lock_bh(&br->lock);
        p->state = BR_STATE_FORWARDING;
        p->flags |= BR_MRP_AWARE;
        spin_unlock_bh(&br->lock);
        rcu_assign_pointer(mrp->i_port, p);

        mrp->in_role = role->in_role;
        mrp->in_id = role->in_id;

        /* If there is an error just bailed out */
        support = br_mrp_switchdev_set_in_role(br, mrp, role->in_id,
                                               role->ring_id, role->in_role);
        if (support == BR_MRP_NONE)
                return -EOPNOTSUPP;

        /* Now detect if the HW actually applied the role or not. If the HW
         * applied the role it means that the SW will not to do those operations
         * anymore. For example if the role is MIM then the HW will notify the
         * SW when interconnect ring is open, but if the is not pushed to the HW
         * the SW will need to detect when the interconnect ring is open.
         */
        mrp->in_role_offloaded = support == BR_MRP_SW ? 0 : 1;

        return 0;
}

/* Start to generate MRP_InTest frames, the frames are generated by
 * HW and if it fails, they are generated by the SW.
 * note: already called with rtnl_lock
 */
int br_mrp_start_in_test(struct net_bridge *br,
                         struct br_mrp_start_in_test *in_test)
{
        struct br_mrp *mrp = br_mrp_find_in_id(br, in_test->in_id);
        enum br_mrp_hw_support support;

        if (!mrp)
                return -EINVAL;

        if (mrp->in_role != BR_MRP_IN_ROLE_MIM)
                return -EINVAL;

        /* Try to push it to the HW and if it fails then continue with SW
         * implementation and if that also fails then return error.
         */
        support =  br_mrp_switchdev_send_in_test(br, mrp, in_test->interval,
                                                 in_test->max_miss,
                                                 in_test->period);
        if (support == BR_MRP_NONE)
                return -EOPNOTSUPP;

        if (support == BR_MRP_HW)
                return 0;

        mrp->in_test_interval = in_test->interval;
        mrp->in_test_end = jiffies + usecs_to_jiffies(in_test->period);
        mrp->in_test_max_miss = in_test->max_miss;
        mrp->in_test_count_miss = 0;
        queue_delayed_work(system_wq, &mrp->in_test_work,
                           usecs_to_jiffies(in_test->interval));

        return 0;
}

/* Determine if the frame type is a ring frame */
static bool br_mrp_ring_frame(struct sk_buff *skb)
{
        const struct br_mrp_tlv_hdr *hdr;
        struct br_mrp_tlv_hdr _hdr;

        hdr = skb_header_pointer(skb, sizeof(uint16_t), sizeof(_hdr), &_hdr);
        if (!hdr)
                return false;

        if (hdr->type == BR_MRP_TLV_HEADER_RING_TEST ||
            hdr->type == BR_MRP_TLV_HEADER_RING_TOPO ||
            hdr->type == BR_MRP_TLV_HEADER_RING_LINK_DOWN ||
            hdr->type == BR_MRP_TLV_HEADER_RING_LINK_UP ||
            hdr->type == BR_MRP_TLV_HEADER_OPTION)
                return true;

        return false;
}

/* Determine if the frame type is an interconnect frame */
static bool br_mrp_in_frame(struct sk_buff *skb)
{
        const struct br_mrp_tlv_hdr *hdr;
        struct br_mrp_tlv_hdr _hdr;

        hdr = skb_header_pointer(skb, sizeof(uint16_t), sizeof(_hdr), &_hdr);
        if (!hdr)
                return false;

        if (hdr->type == BR_MRP_TLV_HEADER_IN_TEST ||
            hdr->type == BR_MRP_TLV_HEADER_IN_TOPO ||
            hdr->type == BR_MRP_TLV_HEADER_IN_LINK_DOWN ||
            hdr->type == BR_MRP_TLV_HEADER_IN_LINK_UP ||
            hdr->type == BR_MRP_TLV_HEADER_IN_LINK_STATUS)
                return true;

        return false;
}

/* Process only MRP Test frame. All the other MRP frames are processed by
 * userspace application
 * note: already called with rcu_read_lock
 */
static void br_mrp_mrm_process(struct br_mrp *mrp, struct net_bridge_port *port,
                               struct sk_buff *skb)
{
        const struct br_mrp_tlv_hdr *hdr;
        struct br_mrp_tlv_hdr _hdr;

        /* Each MRP header starts with a version field which is 16 bits.
         * Therefore skip the version and get directly the TLV header.
         */
        hdr = skb_header_pointer(skb, sizeof(uint16_t), sizeof(_hdr), &_hdr);
        if (!hdr)
                return;

        if (hdr->type != BR_MRP_TLV_HEADER_RING_TEST)
                return;

        mrp->test_count_miss = 0;

        /* Notify the userspace that the ring is closed only when the ring is
         * not closed
         */
        if (mrp->ring_state != BR_MRP_RING_STATE_CLOSED)
                br_mrp_ring_port_open(port->dev, false);
}

/* Determine if the test hdr has a better priority than the node */
static bool br_mrp_test_better_than_own(struct br_mrp *mrp,
                                        struct net_bridge *br,
                                        const struct br_mrp_ring_test_hdr *hdr)
{
        u16 prio = be16_to_cpu(hdr->prio);

        if (prio < mrp->prio ||
            (prio == mrp->prio &&
            ether_addr_to_u64(hdr->sa) < ether_addr_to_u64(br->dev->dev_addr)))
                return true;

        return false;
}

/* Process only MRP Test frame. All the other MRP frames are processed by
 * userspace application
 * note: already called with rcu_read_lock
 */
static void br_mrp_mra_process(struct br_mrp *mrp, struct net_bridge *br,
                               struct net_bridge_port *port,
                               struct sk_buff *skb)
{
        const struct br_mrp_ring_test_hdr *test_hdr;
        struct br_mrp_ring_test_hdr _test_hdr;
        const struct br_mrp_tlv_hdr *hdr;
        struct br_mrp_tlv_hdr _hdr;

        /* Each MRP header starts with a version field which is 16 bits.
         * Therefore skip the version and get directly the TLV header.
         */
        hdr = skb_header_pointer(skb, sizeof(uint16_t), sizeof(_hdr), &_hdr);
        if (!hdr)
                return;

        if (hdr->type != BR_MRP_TLV_HEADER_RING_TEST)
                return;

        test_hdr = skb_header_pointer(skb, sizeof(uint16_t) + sizeof(_hdr),
                                      sizeof(_test_hdr), &_test_hdr);
        if (!test_hdr)
                return;

        /* Only frames that have a better priority than the node will
         * clear the miss counter because otherwise the node will need to behave
         * as MRM.
         */
        if (br_mrp_test_better_than_own(mrp, br, test_hdr))
                mrp->test_count_miss = 0;
}

/* Process only MRP InTest frame. All the other MRP frames are processed by
 * userspace application
 * note: already called with rcu_read_lock
 */
static bool br_mrp_mim_process(struct br_mrp *mrp, struct net_bridge_port *port,
                               struct sk_buff *skb)
{
        const struct br_mrp_in_test_hdr *in_hdr;
        struct br_mrp_in_test_hdr _in_hdr;
        const struct br_mrp_tlv_hdr *hdr;
        struct br_mrp_tlv_hdr _hdr;

        /* Each MRP header starts with a version field which is 16 bits.
         * Therefore skip the version and get directly the TLV header.
         */
        hdr = skb_header_pointer(skb, sizeof(uint16_t), sizeof(_hdr), &_hdr);
        if (!hdr)
                return false;

        /* The check for InTest frame type was already done */
        in_hdr = skb_header_pointer(skb, sizeof(uint16_t) + sizeof(_hdr),
                                    sizeof(_in_hdr), &_in_hdr);
        if (!in_hdr)
                return false;

        /* It needs to process only it's own InTest frames. */
        if (mrp->in_id != ntohs(in_hdr->id))
                return false;

        mrp->in_test_count_miss = 0;

        /* Notify the userspace that the ring is closed only when the ring is
         * not closed
         */
        if (mrp->in_state != BR_MRP_IN_STATE_CLOSED)
                br_mrp_in_port_open(port->dev, false);

        return true;
}

/* Get the MRP frame type
 * note: already called with rcu_read_lock
 */
static u8 br_mrp_get_frame_type(struct sk_buff *skb)
{
        const struct br_mrp_tlv_hdr *hdr;
        struct br_mrp_tlv_hdr _hdr;

        /* Each MRP header starts with a version field which is 16 bits.
         * Therefore skip the version and get directly the TLV header.
         */
        hdr = skb_header_pointer(skb, sizeof(uint16_t), sizeof(_hdr), &_hdr);
        if (!hdr)
                return 0xff;

        return hdr->type;
}

static bool br_mrp_mrm_behaviour(struct br_mrp *mrp)
{
        if (mrp->ring_role == BR_MRP_RING_ROLE_MRM ||
            (mrp->ring_role == BR_MRP_RING_ROLE_MRA && !mrp->test_monitor))
                return true;

        return false;
}

static bool br_mrp_mrc_behaviour(struct br_mrp *mrp)
{
        if (mrp->ring_role == BR_MRP_RING_ROLE_MRC ||
            (mrp->ring_role == BR_MRP_RING_ROLE_MRA && mrp->test_monitor))
                return true;

        return false;
}

/* This will just forward the frame to the other mrp ring ports, depending on
 * the frame type, ring role and interconnect role
 * note: already called with rcu_read_lock
 */
static int br_mrp_rcv(struct net_bridge_port *p,
                      struct sk_buff *skb, struct net_device *dev)
{
        struct net_bridge_port *p_port, *s_port, *i_port = NULL;
        struct net_bridge_port *p_dst, *s_dst, *i_dst = NULL;
        struct net_bridge *br;
        struct br_mrp *mrp;

        /* If port is disabled don't accept any frames */
        if (p->state == BR_STATE_DISABLED)
                return 0;

        br = p->br;
        mrp =  br_mrp_find_port(br, p);
        if (unlikely(!mrp))
                return 0;

        p_port = rcu_dereference(mrp->p_port);
        if (!p_port)
                return 0;
        p_dst = p_port;

        s_port = rcu_dereference(mrp->s_port);
        if (!s_port)
                return 0;
        s_dst = s_port;

        /* If the frame is a ring frame then it is not required to check the
         * interconnect role and ports to process or forward the frame
         */
        if (br_mrp_ring_frame(skb)) {
                /* If the role is MRM then don't forward the frames */
                if (mrp->ring_role == BR_MRP_RING_ROLE_MRM) {
                        br_mrp_mrm_process(mrp, p, skb);
                        goto no_forward;
                }

                /* If the role is MRA then don't forward the frames if it
                 * behaves as MRM node
                 */
                if (mrp->ring_role == BR_MRP_RING_ROLE_MRA) {
                        if (!mrp->test_monitor) {
                                br_mrp_mrm_process(mrp, p, skb);
                                goto no_forward;
                        }

                        br_mrp_mra_process(mrp, br, p, skb);
                }

                goto forward;
        }

        if (br_mrp_in_frame(skb)) {
                u8 in_type = br_mrp_get_frame_type(skb);

                i_port = rcu_dereference(mrp->i_port);
                i_dst = i_port;

                /* If the ring port is in block state it should not forward
                 * In_Test frames
                 */
                if (br_mrp_is_ring_port(p_port, s_port, p) &&
                    p->state == BR_STATE_BLOCKING &&
                    in_type == BR_MRP_TLV_HEADER_IN_TEST)
                        goto no_forward;

                /* Nodes that behaves as MRM needs to stop forwarding the
                 * frames in case the ring is closed, otherwise will be a loop.
                 * In this case the frame is no forward between the ring ports.
                 */
                if (br_mrp_mrm_behaviour(mrp) &&
                    br_mrp_is_ring_port(p_port, s_port, p) &&
                    (s_port->state != BR_STATE_FORWARDING ||
                     p_port->state != BR_STATE_FORWARDING)) {
                        p_dst = NULL;
                        s_dst = NULL;
                }

                /* A node that behaves as MRC and doesn't have a interconnect
                 * role then it should forward all frames between the ring ports
                 * because it doesn't have an interconnect port
                 */
                if (br_mrp_mrc_behaviour(mrp) &&
                    mrp->in_role == BR_MRP_IN_ROLE_DISABLED)
                        goto forward;

                if (mrp->in_role == BR_MRP_IN_ROLE_MIM) {
                        if (in_type == BR_MRP_TLV_HEADER_IN_TEST) {
                                /* MIM should not forward it's own InTest
                                 * frames
                                 */
                                if (br_mrp_mim_process(mrp, p, skb)) {
                                        goto no_forward;
                                } else {
                                        if (br_mrp_is_ring_port(p_port, s_port,
                                                                p))
                                                i_dst = NULL;

                                        if (br_mrp_is_in_port(i_port, p))
                                                goto no_forward;
                                }
                        } else {
                                /* MIM should forward IntLinkChange/Status and
                                 * IntTopoChange between ring ports but MIM
                                 * should not forward IntLinkChange/Status and
                                 * IntTopoChange if the frame was received at
                                 * the interconnect port
                                 */
                                if (br_mrp_is_ring_port(p_port, s_port, p))
                                        i_dst = NULL;

                                if (br_mrp_is_in_port(i_port, p))
                                        goto no_forward;
                        }
                }

                if (mrp->in_role == BR_MRP_IN_ROLE_MIC) {
                        /* MIC should forward InTest frames on all ports
                         * regardless of the received port
                         */
                        if (in_type == BR_MRP_TLV_HEADER_IN_TEST)
                                goto forward;

                        /* MIC should forward IntLinkChange frames only if they
                         * are received on ring ports to all the ports
                         */
                        if (br_mrp_is_ring_port(p_port, s_port, p) &&
                            (in_type == BR_MRP_TLV_HEADER_IN_LINK_UP ||
                             in_type == BR_MRP_TLV_HEADER_IN_LINK_DOWN))
                                goto forward;

                        /* MIC should forward IntLinkStatus frames only to
                         * interconnect port if it was received on a ring port.
                         * If it is received on interconnect port then, it
                         * should be forward on both ring ports
                         */
                        if (br_mrp_is_ring_port(p_port, s_port, p) &&
                            in_type == BR_MRP_TLV_HEADER_IN_LINK_STATUS) {
                                p_dst = NULL;
                                s_dst = NULL;
                        }

                        /* Should forward the InTopo frames only between the
                         * ring ports
                         */
                        if (in_type == BR_MRP_TLV_HEADER_IN_TOPO) {
                                i_dst = NULL;
                                goto forward;
                        }

                        /* In all the other cases don't forward the frames */
                        goto no_forward;
                }
        }

forward:
        if (p_dst)
                br_forward(p_dst, skb, true, false);
        if (s_dst)
                br_forward(s_dst, skb, true, false);
        if (i_dst)
                br_forward(i_dst, skb, true, false);

no_forward:
        return 1;
}

/* Check if the frame was received on a port that is part of MRP ring
 * and if the frame has MRP eth. In that case process the frame otherwise do
 * normal forwarding.
 * note: already called with rcu_read_lock
 */
static int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb)
{
        /* If there is no MRP instance do normal forwarding */
        if (likely(!(p->flags & BR_MRP_AWARE)))
                goto out;

        return br_mrp_rcv(p, skb, p->dev);
out:
        return 0;
}

bool br_mrp_enabled(struct net_bridge *br)
{
        return !hlist_empty(&br->mrp_list);
}






























































 8772 

























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_GENERIC_DELAY_H
#define __ASM_GENERIC_DELAY_H

#include <linux/math.h>
#include <vdso/time64.h>

/* Undefined functions to get compile-time errors */
extern void __bad_udelay(void);
extern void __bad_ndelay(void);

extern void __udelay(unsigned long usecs);
extern void __ndelay(unsigned long nsecs);
extern void __const_udelay(unsigned long xloops);
extern void __delay(unsigned long loops);

/*
 * The microseconds/nanosecond delay multiplicators are used to convert a
 * constant microseconds/nanoseconds value to a value which can be used by the
 * architectures specific implementation to transform it into loops.
 */
#define UDELAY_CONST_MULT        ((unsigned long)DIV_ROUND_UP(1ULL << 32, USEC_PER_SEC))
#define NDELAY_CONST_MULT        ((unsigned long)DIV_ROUND_UP(1ULL << 32, NSEC_PER_SEC))

/*
 * The maximum constant udelay/ndelay value picked out of thin air to prevent
 * too long constant udelays/ndelays.
 */
#define DELAY_CONST_MAX   20000

/**
 * udelay - Inserting a delay based on microseconds with busy waiting
 * @usec:        requested delay in microseconds
 *
 * When delaying in an atomic context ndelay(), udelay() and mdelay() are the
 * only valid variants of delaying/sleeping to go with.
 *
 * When inserting delays in non atomic context which are shorter than the time
 * which is required to queue e.g. an hrtimer and to enter then the scheduler,
 * it is also valuable to use udelay(). But it is not simple to specify a
 * generic threshold for this which will fit for all systems. An approximation
 * is a threshold for all delays up to 10 microseconds.
 *
 * When having a delay which is larger than the architecture specific
 * %MAX_UDELAY_MS value, please make sure mdelay() is used. Otherwise a overflow
 * risk is given.
 *
 * Please note that ndelay(), udelay() and mdelay() may return early for several
 * reasons (https://lists.openwall.net/linux-kernel/2011/01/09/56):
 *
 * #. computed loops_per_jiffy too low (due to the time taken to execute the
 *    timer interrupt.)
 * #. cache behaviour affecting the time it takes to execute the loop function.
 * #. CPU clock rate changes.
 */
static __always_inline void udelay(unsigned long usec)
{
        if (__builtin_constant_p(usec)) {
                if (usec >= DELAY_CONST_MAX)
                        __bad_udelay();
                else
                        __const_udelay(usec * UDELAY_CONST_MULT);
        } else {
                __udelay(usec);
        }
}

/**
 * ndelay - Inserting a delay based on nanoseconds with busy waiting
 * @nsec:        requested delay in nanoseconds
 *
 * See udelay() for basic information about ndelay() and it's variants.
 */
static __always_inline void ndelay(unsigned long nsec)
{
        if (__builtin_constant_p(nsec)) {
                if (nsec >= DELAY_CONST_MAX)
                        __bad_ndelay();
                else
                        __const_udelay(nsec * NDELAY_CONST_MULT);
        } else {
                __ndelay(nsec);
        }
}
#define ndelay(x) ndelay(x)

#endif /* __ASM_GENERIC_DELAY_H */






























































































































    5 
































    9 













    5 




























   12 









   12 






















































































    9 



    9 

    9 





    9 


    9 









    9 





    9 
    9 
























    5 




    5 










































































    9 







    9 
    9 

    9 




    5 




    9 














    8 

    8 
    8 










    5 
    5 

    5 


















    8 


    7 




    7 


    4 
    4 


    2 
    2 


    2 






    8 
    5 

    8 




    5 
    3 








    5 









    5 







    5 



    5 





    5 


    2 

    5 











    4 






























































































    3 



    3 
    3 

    3 




    3 








    3 

































































    9 



    9 






















































    5 










    5 
    5 


    5 


    5 
    5 


















    9 
    9 







    9 
    9 
    9 








    9 
    9 
    9 



    9 
    8 





















    9 




    9 

























    4 
    4 

    4 


    4 




    4 




    4 



    4 






































































    3 








    3 
    3 

    3 



    3 


    3 

    3 
















    1 








    1 
    1 

    1 







    1 


    1 
    1 
    1 













    1 
























































    4 





    4 
    4 

    4 





















































































































































































































































    4 









    4 
    4 

    4 







    4 


    4 


    4 



    4 





    4 













    9 


    9 



    9 





    9 
    4 





    9 


    5 







    5 

    5 

    5 









    9 





    9 






    9 



    9 






    9 





































    4 



















    4 



    4 
    4 




    4 



    4 








    9 
    9 


    9 
    9 



















    9 
    9 

    9 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
/*
 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
 * Copyright (c) 2005 Intel Corporation. All rights reserved.
 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
 * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/if_vlan.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/netdevice.h>
#include <net/addrconf.h>

#include <rdma/ib_cache.h>

#include "core_priv.h"

struct ib_pkey_cache {
        int             table_len;
        u16             table[] __counted_by(table_len);
};

struct ib_update_work {
        struct work_struct work;
        struct ib_event event;
        bool enforce_security;
};

union ib_gid zgid;
EXPORT_SYMBOL(zgid);

enum gid_attr_find_mask {
        GID_ATTR_FIND_MASK_GID          = 1UL << 0,
        GID_ATTR_FIND_MASK_NETDEV        = 1UL << 1,
        GID_ATTR_FIND_MASK_DEFAULT        = 1UL << 2,
        GID_ATTR_FIND_MASK_GID_TYPE        = 1UL << 3,
};

enum gid_table_entry_state {
        GID_TABLE_ENTRY_INVALID                = 1,
        GID_TABLE_ENTRY_VALID                = 2,
        /*
         * Indicates that entry is pending to be removed, there may
         * be active users of this GID entry.
         * When last user of the GID entry releases reference to it,
         * GID entry is detached from the table.
         */
        GID_TABLE_ENTRY_PENDING_DEL        = 3,
};

struct roce_gid_ndev_storage {
        struct rcu_head rcu_head;
        struct net_device *ndev;
};

struct ib_gid_table_entry {
        struct kref                        kref;
        struct work_struct                del_work;
        struct ib_gid_attr                attr;
        void                                *context;
        /* Store the ndev pointer to release reference later on in
         * call_rcu context because by that time gid_table_entry
         * and attr might be already freed. So keep a copy of it.
         * ndev_storage is freed by rcu callback.
         */
        struct roce_gid_ndev_storage        *ndev_storage;
        enum gid_table_entry_state        state;
};

struct ib_gid_table {
        int                                sz;
        /* In RoCE, adding a GID to the table requires:
         * (a) Find if this GID is already exists.
         * (b) Find a free space.
         * (c) Write the new GID
         *
         * Delete requires different set of operations:
         * (a) Find the GID
         * (b) Delete it.
         *
         **/
        /* Any writer to data_vec must hold this lock and the write side of
         * rwlock. Readers must hold only rwlock. All writers must be in a
         * sleepable context.
         */
        struct mutex                        lock;
        /* rwlock protects data_vec[ix]->state and entry pointer.
         */
        rwlock_t                        rwlock;
        struct ib_gid_table_entry        **data_vec;
        /* bit field, each bit indicates the index of default GID */
        u32                                default_gid_indices;
};

static void dispatch_gid_change_event(struct ib_device *ib_dev, u32 port)
{
        struct ib_event event;

        event.device                = ib_dev;
        event.element.port_num        = port;
        event.event                = IB_EVENT_GID_CHANGE;

        ib_dispatch_event_clients(&event);
}

static const char * const gid_type_str[] = {
        /* IB/RoCE v1 value is set for IB_GID_TYPE_IB and IB_GID_TYPE_ROCE for
         * user space compatibility reasons.
         */
        [IB_GID_TYPE_IB]        = "IB/RoCE v1",
        [IB_GID_TYPE_ROCE]        = "IB/RoCE v1",
        [IB_GID_TYPE_ROCE_UDP_ENCAP]        = "RoCE v2",
};

const char *ib_cache_gid_type_str(enum ib_gid_type gid_type)
{
        if (gid_type < ARRAY_SIZE(gid_type_str) && gid_type_str[gid_type])
                return gid_type_str[gid_type];

        return "Invalid GID type";
}
EXPORT_SYMBOL(ib_cache_gid_type_str);

/** rdma_is_zero_gid - Check if given GID is zero or not.
 * @gid:        GID to check
 * Returns true if given GID is zero, returns false otherwise.
 */
bool rdma_is_zero_gid(const union ib_gid *gid)
{
        return !memcmp(gid, &zgid, sizeof(*gid));
}
EXPORT_SYMBOL(rdma_is_zero_gid);

/** is_gid_index_default - Check if a given index belongs to
 * reserved default GIDs or not.
 * @table:        GID table pointer
 * @index:        Index to check in GID table
 * Returns true if index is one of the reserved default GID index otherwise
 * returns false.
 */
static bool is_gid_index_default(const struct ib_gid_table *table,
                                 unsigned int index)
{
        return index < 32 && (BIT(index) & table->default_gid_indices);
}

int ib_cache_gid_parse_type_str(const char *buf)
{
        unsigned int i;
        size_t len;
        int err = -EINVAL;

        len = strlen(buf);
        if (len == 0)
                return -EINVAL;

        if (buf[len - 1] == '\n')
                len--;

        for (i = 0; i < ARRAY_SIZE(gid_type_str); ++i)
                if (gid_type_str[i] && !strncmp(buf, gid_type_str[i], len) &&
                    len == strlen(gid_type_str[i])) {
                        err = i;
                        break;
                }

        return err;
}
EXPORT_SYMBOL(ib_cache_gid_parse_type_str);

static struct ib_gid_table *rdma_gid_table(struct ib_device *device, u32 port)
{
        return device->port_data[port].cache.gid;
}

static bool is_gid_entry_free(const struct ib_gid_table_entry *entry)
{
        return !entry;
}

static bool is_gid_entry_valid(const struct ib_gid_table_entry *entry)
{
        return entry && entry->state == GID_TABLE_ENTRY_VALID;
}

static void schedule_free_gid(struct kref *kref)
{
        struct ib_gid_table_entry *entry =
                        container_of(kref, struct ib_gid_table_entry, kref);

        queue_work(ib_wq, &entry->del_work);
}

static void put_gid_ndev(struct rcu_head *head)
{
        struct roce_gid_ndev_storage *storage =
                container_of(head, struct roce_gid_ndev_storage, rcu_head);

        WARN_ON(!storage->ndev);
        /* At this point its safe to release netdev reference,
         * as all callers working on gid_attr->ndev are done
         * using this netdev.
         */
        dev_put(storage->ndev);
        kfree(storage);
}

static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
{
        struct ib_device *device = entry->attr.device;
        u32 port_num = entry->attr.port_num;
        struct ib_gid_table *table = rdma_gid_table(device, port_num);

        dev_dbg(&device->dev, "%s port=%u index=%u gid %pI6\n", __func__,
                port_num, entry->attr.index, entry->attr.gid.raw);

        write_lock_irq(&table->rwlock);

        /*
         * The only way to avoid overwriting NULL in table is
         * by comparing if it is same entry in table or not!
         * If new entry in table is added by the time we free here,
         * don't overwrite the table entry.
         */
        if (entry == table->data_vec[entry->attr.index])
                table->data_vec[entry->attr.index] = NULL;
        /* Now this index is ready to be allocated */
        write_unlock_irq(&table->rwlock);

        if (entry->ndev_storage)
                call_rcu(&entry->ndev_storage->rcu_head, put_gid_ndev);
        kfree(entry);
}

static void free_gid_entry(struct kref *kref)
{
        struct ib_gid_table_entry *entry =
                        container_of(kref, struct ib_gid_table_entry, kref);

        free_gid_entry_locked(entry);
}

/**
 * free_gid_work - Release reference to the GID entry
 * @work: Work structure to refer to GID entry which needs to be
 * deleted.
 *
 * free_gid_work() frees the entry from the HCA's hardware table
 * if provider supports it. It releases reference to netdevice.
 */
static void free_gid_work(struct work_struct *work)
{
        struct ib_gid_table_entry *entry =
                container_of(work, struct ib_gid_table_entry, del_work);
        struct ib_device *device = entry->attr.device;
        u32 port_num = entry->attr.port_num;
        struct ib_gid_table *table = rdma_gid_table(device, port_num);

        mutex_lock(&table->lock);
        free_gid_entry_locked(entry);
        mutex_unlock(&table->lock);
}

static struct ib_gid_table_entry *
alloc_gid_entry(const struct ib_gid_attr *attr)
{
        struct ib_gid_table_entry *entry;
        struct net_device *ndev;

        entry = kzalloc(sizeof(*entry), GFP_KERNEL);
        if (!entry)
                return NULL;

        ndev = rcu_dereference_protected(attr->ndev, 1);
        if (ndev) {
                entry->ndev_storage = kzalloc(sizeof(*entry->ndev_storage),
                                              GFP_KERNEL);
                if (!entry->ndev_storage) {
                        kfree(entry);
                        return NULL;
                }
                dev_hold(ndev);
                entry->ndev_storage->ndev = ndev;
        }
        kref_init(&entry->kref);
        memcpy(&entry->attr, attr, sizeof(*attr));
        INIT_WORK(&entry->del_work, free_gid_work);
        entry->state = GID_TABLE_ENTRY_INVALID;
        return entry;
}

static void store_gid_entry(struct ib_gid_table *table,
                            struct ib_gid_table_entry *entry)
{
        entry->state = GID_TABLE_ENTRY_VALID;

        dev_dbg(&entry->attr.device->dev, "%s port=%u index=%u gid %pI6\n",
                __func__, entry->attr.port_num, entry->attr.index,
                entry->attr.gid.raw);

        lockdep_assert_held(&table->lock);
        write_lock_irq(&table->rwlock);
        table->data_vec[entry->attr.index] = entry;
        write_unlock_irq(&table->rwlock);
}

static void get_gid_entry(struct ib_gid_table_entry *entry)
{
        kref_get(&entry->kref);
}

static void put_gid_entry(struct ib_gid_table_entry *entry)
{
        kref_put(&entry->kref, schedule_free_gid);
}

static void put_gid_entry_locked(struct ib_gid_table_entry *entry)
{
        kref_put(&entry->kref, free_gid_entry);
}

static int add_roce_gid(struct ib_gid_table_entry *entry)
{
        const struct ib_gid_attr *attr = &entry->attr;
        int ret;

        if (!attr->ndev) {
                dev_err(&attr->device->dev, "%s NULL netdev port=%u index=%u\n",
                        __func__, attr->port_num, attr->index);
                return -EINVAL;
        }
        if (rdma_cap_roce_gid_table(attr->device, attr->port_num)) {
                ret = attr->device->ops.add_gid(attr, &entry->context);
                if (ret) {
                        dev_err(&attr->device->dev,
                                "%s GID add failed port=%u index=%u\n",
                                __func__, attr->port_num, attr->index);
                        return ret;
                }
        }
        return 0;
}

/**
 * del_gid - Delete GID table entry
 *
 * @ib_dev:        IB device whose GID entry to be deleted
 * @port:        Port number of the IB device
 * @table:        GID table of the IB device for a port
 * @ix:                GID entry index to delete
 *
 */
static void del_gid(struct ib_device *ib_dev, u32 port,
                    struct ib_gid_table *table, int ix)
{
        struct roce_gid_ndev_storage *ndev_storage;
        struct ib_gid_table_entry *entry;

        lockdep_assert_held(&table->lock);

        dev_dbg(&ib_dev->dev, "%s port=%u index=%d gid %pI6\n", __func__, port,
                ix, table->data_vec[ix]->attr.gid.raw);

        write_lock_irq(&table->rwlock);
        entry = table->data_vec[ix];
        entry->state = GID_TABLE_ENTRY_PENDING_DEL;
        /*
         * For non RoCE protocol, GID entry slot is ready to use.
         */
        if (!rdma_protocol_roce(ib_dev, port))
                table->data_vec[ix] = NULL;
        write_unlock_irq(&table->rwlock);

        if (rdma_cap_roce_gid_table(ib_dev, port))
                ib_dev->ops.del_gid(&entry->attr, &entry->context);

        ndev_storage = entry->ndev_storage;
        if (ndev_storage) {
                entry->ndev_storage = NULL;
                rcu_assign_pointer(entry->attr.ndev, NULL);
                call_rcu(&ndev_storage->rcu_head, put_gid_ndev);
        }

        put_gid_entry_locked(entry);
}

/**
 * add_modify_gid - Add or modify GID table entry
 *
 * @table:        GID table in which GID to be added or modified
 * @attr:        Attributes of the GID
 *
 * Returns 0 on success or appropriate error code. It accepts zero
 * GID addition for non RoCE ports for HCA's who report them as valid
 * GID. However such zero GIDs are not added to the cache.
 */
static int add_modify_gid(struct ib_gid_table *table,
                          const struct ib_gid_attr *attr)
{
        struct ib_gid_table_entry *entry;
        int ret = 0;

        /*
         * Invalidate any old entry in the table to make it safe to write to
         * this index.
         */
        if (is_gid_entry_valid(table->data_vec[attr->index]))
                del_gid(attr->device, attr->port_num, table, attr->index);

        /*
         * Some HCA's report multiple GID entries with only one valid GID, and
         * leave other unused entries as the zero GID. Convert zero GIDs to
         * empty table entries instead of storing them.
         */
        if (rdma_is_zero_gid(&attr->gid))
                return 0;

        entry = alloc_gid_entry(attr);
        if (!entry)
                return -ENOMEM;

        if (rdma_protocol_roce(attr->device, attr->port_num)) {
                ret = add_roce_gid(entry);
                if (ret)
                        goto done;
        }

        store_gid_entry(table, entry);
        return 0;

done:
        put_gid_entry(entry);
        return ret;
}

/* rwlock should be read locked, or lock should be held */
static int find_gid(struct ib_gid_table *table, const union ib_gid *gid,
                    const struct ib_gid_attr *val, bool default_gid,
                    unsigned long mask, int *pempty)
{
        int i = 0;
        int found = -1;
        int empty = pempty ? -1 : 0;

        while (i < table->sz && (found < 0 || empty < 0)) {
                struct ib_gid_table_entry *data = table->data_vec[i];
                struct ib_gid_attr *attr;
                int curr_index = i;

                i++;

                /* find_gid() is used during GID addition where it is expected
                 * to return a free entry slot which is not duplicate.
                 * Free entry slot is requested and returned if pempty is set,
                 * so lookup free slot only if requested.
                 */
                if (pempty && empty < 0) {
                        if (is_gid_entry_free(data) &&
                            default_gid ==
                                is_gid_index_default(table, curr_index)) {
                                /*
                                 * Found an invalid (free) entry; allocate it.
                                 * If default GID is requested, then our
                                 * found slot must be one of the DEFAULT
                                 * reserved slots or we fail.
                                 * This ensures that only DEFAULT reserved
                                 * slots are used for default property GIDs.
                                 */
                                empty = curr_index;
                        }
                }

                /*
                 * Additionally find_gid() is used to find valid entry during
                 * lookup operation; so ignore the entries which are marked as
                 * pending for removal and the entries which are marked as
                 * invalid.
                 */
                if (!is_gid_entry_valid(data))
                        continue;

                if (found >= 0)
                        continue;

                attr = &data->attr;
                if (mask & GID_ATTR_FIND_MASK_GID_TYPE &&
                    attr->gid_type != val->gid_type)
                        continue;

                if (mask & GID_ATTR_FIND_MASK_GID &&
                    memcmp(gid, &data->attr.gid, sizeof(*gid)))
                        continue;

                if (mask & GID_ATTR_FIND_MASK_NETDEV &&
                    attr->ndev != val->ndev)
                        continue;

                if (mask & GID_ATTR_FIND_MASK_DEFAULT &&
                    is_gid_index_default(table, curr_index) != default_gid)
                        continue;

                found = curr_index;
        }

        if (pempty)
                *pempty = empty;

        return found;
}

static void make_default_gid(struct  net_device *dev, union ib_gid *gid)
{
        gid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
        addrconf_ifid_eui48(&gid->raw[8], dev);
}

static int __ib_cache_gid_add(struct ib_device *ib_dev, u32 port,
                              union ib_gid *gid, struct ib_gid_attr *attr,
                              unsigned long mask, bool default_gid)
{
        struct ib_gid_table *table;
        int ret = 0;
        int empty;
        int ix;

        /* Do not allow adding zero GID in support of
         * IB spec version 1.3 section 4.1.1 point (6) and
         * section 12.7.10 and section 12.7.20
         */
        if (rdma_is_zero_gid(gid))
                return -EINVAL;

        table = rdma_gid_table(ib_dev, port);

        mutex_lock(&table->lock);

        ix = find_gid(table, gid, attr, default_gid, mask, &empty);
        if (ix >= 0)
                goto out_unlock;

        if (empty < 0) {
                ret = -ENOSPC;
                goto out_unlock;
        }
        attr->device = ib_dev;
        attr->index = empty;
        attr->port_num = port;
        attr->gid = *gid;
        ret = add_modify_gid(table, attr);
        if (!ret)
                dispatch_gid_change_event(ib_dev, port);

out_unlock:
        mutex_unlock(&table->lock);
        if (ret)
                pr_warn("%s: unable to add gid %pI6 error=%d\n",
                        __func__, gid->raw, ret);
        return ret;
}

int ib_cache_gid_add(struct ib_device *ib_dev, u32 port,
                     union ib_gid *gid, struct ib_gid_attr *attr)
{
        unsigned long mask = GID_ATTR_FIND_MASK_GID |
                             GID_ATTR_FIND_MASK_GID_TYPE |
                             GID_ATTR_FIND_MASK_NETDEV;

        return __ib_cache_gid_add(ib_dev, port, gid, attr, mask, false);
}

static int
_ib_cache_gid_del(struct ib_device *ib_dev, u32 port,
                  union ib_gid *gid, struct ib_gid_attr *attr,
                  unsigned long mask, bool default_gid)
{
        struct ib_gid_table *table;
        int ret = 0;
        int ix;

        table = rdma_gid_table(ib_dev, port);

        mutex_lock(&table->lock);

        ix = find_gid(table, gid, attr, default_gid, mask, NULL);
        if (ix < 0) {
                ret = -EINVAL;
                goto out_unlock;
        }

        del_gid(ib_dev, port, table, ix);
        dispatch_gid_change_event(ib_dev, port);

out_unlock:
        mutex_unlock(&table->lock);
        if (ret)
                pr_debug("%s: can't delete gid %pI6 error=%d\n",
                         __func__, gid->raw, ret);
        return ret;
}

int ib_cache_gid_del(struct ib_device *ib_dev, u32 port,
                     union ib_gid *gid, struct ib_gid_attr *attr)
{
        unsigned long mask = GID_ATTR_FIND_MASK_GID          |
                             GID_ATTR_FIND_MASK_GID_TYPE |
                             GID_ATTR_FIND_MASK_DEFAULT  |
                             GID_ATTR_FIND_MASK_NETDEV;

        return _ib_cache_gid_del(ib_dev, port, gid, attr, mask, false);
}

int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u32 port,
                                     struct net_device *ndev)
{
        struct ib_gid_table *table;
        int ix;
        bool deleted = false;

        table = rdma_gid_table(ib_dev, port);

        mutex_lock(&table->lock);

        for (ix = 0; ix < table->sz; ix++) {
                if (is_gid_entry_valid(table->data_vec[ix]) &&
                    table->data_vec[ix]->attr.ndev == ndev) {
                        del_gid(ib_dev, port, table, ix);
                        deleted = true;
                }
        }

        mutex_unlock(&table->lock);

        if (deleted)
                dispatch_gid_change_event(ib_dev, port);

        return 0;
}

/**
 * rdma_find_gid_by_port - Returns the GID entry attributes when it finds
 * a valid GID entry for given search parameters. It searches for the specified
 * GID value in the local software cache.
 * @ib_dev: The device to query.
 * @gid: The GID value to search for.
 * @gid_type: The GID type to search for.
 * @port: The port number of the device where the GID value should be searched.
 * @ndev: In RoCE, the net device of the device. NULL means ignore.
 *
 * Returns sgid attributes if the GID is found with valid reference or
 * returns ERR_PTR for the error.
 * The caller must invoke rdma_put_gid_attr() to release the reference.
 */
const struct ib_gid_attr *
rdma_find_gid_by_port(struct ib_device *ib_dev,
                      const union ib_gid *gid,
                      enum ib_gid_type gid_type,
                      u32 port, struct net_device *ndev)
{
        int local_index;
        struct ib_gid_table *table;
        unsigned long mask = GID_ATTR_FIND_MASK_GID |
                             GID_ATTR_FIND_MASK_GID_TYPE;
        struct ib_gid_attr val = {.ndev = ndev, .gid_type = gid_type};
        const struct ib_gid_attr *attr;
        unsigned long flags;

        if (!rdma_is_port_valid(ib_dev, port))
                return ERR_PTR(-ENOENT);

        table = rdma_gid_table(ib_dev, port);

        if (ndev)
                mask |= GID_ATTR_FIND_MASK_NETDEV;

        read_lock_irqsave(&table->rwlock, flags);
        local_index = find_gid(table, gid, &val, false, mask, NULL);
        if (local_index >= 0) {
                get_gid_entry(table->data_vec[local_index]);
                attr = &table->data_vec[local_index]->attr;
                read_unlock_irqrestore(&table->rwlock, flags);
                return attr;
        }

        read_unlock_irqrestore(&table->rwlock, flags);
        return ERR_PTR(-ENOENT);
}
EXPORT_SYMBOL(rdma_find_gid_by_port);

/**
 * rdma_find_gid_by_filter - Returns the GID table attribute where a
 * specified GID value occurs
 * @ib_dev: The device to query.
 * @gid: The GID value to search for.
 * @port: The port number of the device where the GID value could be
 *   searched.
 * @filter: The filter function is executed on any matching GID in the table.
 *   If the filter function returns true, the corresponding index is returned,
 *   otherwise, we continue searching the GID table. It's guaranteed that
 *   while filter is executed, ndev field is valid and the structure won't
 *   change. filter is executed in an atomic context. filter must not be NULL.
 * @context: Private data to pass into the call-back.
 *
 * rdma_find_gid_by_filter() searches for the specified GID value
 * of which the filter function returns true in the port's GID table.
 *
 */
const struct ib_gid_attr *rdma_find_gid_by_filter(
        struct ib_device *ib_dev, const union ib_gid *gid, u32 port,
        bool (*filter)(const union ib_gid *gid, const struct ib_gid_attr *,
                       void *),
        void *context)
{
        const struct ib_gid_attr *res = ERR_PTR(-ENOENT);
        struct ib_gid_table *table;
        unsigned long flags;
        unsigned int i;

        if (!rdma_is_port_valid(ib_dev, port))
                return ERR_PTR(-EINVAL);

        table = rdma_gid_table(ib_dev, port);

        read_lock_irqsave(&table->rwlock, flags);
        for (i = 0; i < table->sz; i++) {
                struct ib_gid_table_entry *entry = table->data_vec[i];

                if (!is_gid_entry_valid(entry))
                        continue;

                if (memcmp(gid, &entry->attr.gid, sizeof(*gid)))
                        continue;

                if (filter(gid, &entry->attr, context)) {
                        get_gid_entry(entry);
                        res = &entry->attr;
                        break;
                }
        }
        read_unlock_irqrestore(&table->rwlock, flags);
        return res;
}

static struct ib_gid_table *alloc_gid_table(int sz)
{
        struct ib_gid_table *table = kzalloc(sizeof(*table), GFP_KERNEL);

        if (!table)
                return NULL;

        table->data_vec = kcalloc(sz, sizeof(*table->data_vec), GFP_KERNEL);
        if (!table->data_vec)
                goto err_free_table;

        mutex_init(&table->lock);

        table->sz = sz;
        rwlock_init(&table->rwlock);
        return table;

err_free_table:
        kfree(table);
        return NULL;
}

static void release_gid_table(struct ib_device *device,
                              struct ib_gid_table *table)
{
        int i;

        if (!table)
                return;

        for (i = 0; i < table->sz; i++) {
                if (is_gid_entry_free(table->data_vec[i]))
                        continue;

                WARN_ONCE(true,
                          "GID entry ref leak for dev %s index %d ref=%u\n",
                          dev_name(&device->dev), i,
                          kref_read(&table->data_vec[i]->kref));
        }

        mutex_destroy(&table->lock);
        kfree(table->data_vec);
        kfree(table);
}

static void cleanup_gid_table_port(struct ib_device *ib_dev, u32 port,
                                   struct ib_gid_table *table)
{
        int i;

        if (!table)
                return;

        mutex_lock(&table->lock);
        for (i = 0; i < table->sz; ++i) {
                if (is_gid_entry_valid(table->data_vec[i]))
                        del_gid(ib_dev, port, table, i);
        }
        mutex_unlock(&table->lock);
}

void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u32 port,
                                  struct net_device *ndev,
                                  unsigned long gid_type_mask,
                                  enum ib_cache_gid_default_mode mode)
{
        union ib_gid gid = { };
        struct ib_gid_attr gid_attr;
        unsigned int gid_type;
        unsigned long mask;

        mask = GID_ATTR_FIND_MASK_GID_TYPE |
               GID_ATTR_FIND_MASK_DEFAULT |
               GID_ATTR_FIND_MASK_NETDEV;
        memset(&gid_attr, 0, sizeof(gid_attr));
        gid_attr.ndev = ndev;

        for (gid_type = 0; gid_type < IB_GID_TYPE_SIZE; ++gid_type) {
                if (1UL << gid_type & ~gid_type_mask)
                        continue;

                gid_attr.gid_type = gid_type;

                if (mode == IB_CACHE_GID_DEFAULT_MODE_SET) {
                        make_default_gid(ndev, &gid);
                        __ib_cache_gid_add(ib_dev, port, &gid,
                                           &gid_attr, mask, true);
                } else if (mode == IB_CACHE_GID_DEFAULT_MODE_DELETE) {
                        _ib_cache_gid_del(ib_dev, port, &gid,
                                          &gid_attr, mask, true);
                }
        }
}

static void gid_table_reserve_default(struct ib_device *ib_dev, u32 port,
                                      struct ib_gid_table *table)
{
        unsigned int i;
        unsigned long roce_gid_type_mask;
        unsigned int num_default_gids;

        roce_gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
        num_default_gids = hweight_long(roce_gid_type_mask);
        /* Reserve starting indices for default GIDs */
        for (i = 0; i < num_default_gids && i < table->sz; i++)
                table->default_gid_indices |= BIT(i);
}


static void gid_table_release_one(struct ib_device *ib_dev)
{
        u32 p;

        rdma_for_each_port (ib_dev, p) {
                release_gid_table(ib_dev, ib_dev->port_data[p].cache.gid);
                ib_dev->port_data[p].cache.gid = NULL;
        }
}

static int _gid_table_setup_one(struct ib_device *ib_dev)
{
        struct ib_gid_table *table;
        u32 rdma_port;

        rdma_for_each_port (ib_dev, rdma_port) {
                table = alloc_gid_table(
                        ib_dev->port_data[rdma_port].immutable.gid_tbl_len);
                if (!table)
                        goto rollback_table_setup;

                gid_table_reserve_default(ib_dev, rdma_port, table);
                ib_dev->port_data[rdma_port].cache.gid = table;
        }
        return 0;

rollback_table_setup:
        gid_table_release_one(ib_dev);
        return -ENOMEM;
}

static void gid_table_cleanup_one(struct ib_device *ib_dev)
{
        u32 p;

        rdma_for_each_port (ib_dev, p)
                cleanup_gid_table_port(ib_dev, p,
                                       ib_dev->port_data[p].cache.gid);
}

static int gid_table_setup_one(struct ib_device *ib_dev)
{
        int err;

        err = _gid_table_setup_one(ib_dev);

        if (err)
                return err;

        rdma_roce_rescan_device(ib_dev);

        return err;
}

/**
 * rdma_query_gid - Read the GID content from the GID software cache
 * @device:                Device to query the GID
 * @port_num:                Port number of the device
 * @index:                Index of the GID table entry to read
 * @gid:                Pointer to GID where to store the entry's GID
 *
 * rdma_query_gid() only reads the GID entry content for requested device,
 * port and index. It reads for IB, RoCE and iWarp link layers.  It doesn't
 * hold any reference to the GID table entry in the HCA or software cache.
 *
 * Returns 0 on success or appropriate error code.
 *
 */
int rdma_query_gid(struct ib_device *device, u32 port_num,
                   int index, union ib_gid *gid)
{
        struct ib_gid_table *table;
        unsigned long flags;
        int res;

        if (!rdma_is_port_valid(device, port_num))
                return -EINVAL;

        table = rdma_gid_table(device, port_num);
        read_lock_irqsave(&table->rwlock, flags);

        if (index < 0 || index >= table->sz) {
                res = -EINVAL;
                goto done;
        }

        if (!is_gid_entry_valid(table->data_vec[index])) {
                res = -ENOENT;
                goto done;
        }

        memcpy(gid, &table->data_vec[index]->attr.gid, sizeof(*gid));
        res = 0;

done:
        read_unlock_irqrestore(&table->rwlock, flags);
        return res;
}
EXPORT_SYMBOL(rdma_query_gid);

/**
 * rdma_read_gid_hw_context - Read the HW GID context from GID attribute
 * @attr:                Potinter to the GID attribute
 *
 * rdma_read_gid_hw_context() reads the drivers GID HW context corresponding
 * to the SGID attr. Callers are required to already be holding the reference
 * to an existing GID entry.
 *
 * Returns the HW GID context
 *
 */
void *rdma_read_gid_hw_context(const struct ib_gid_attr *attr)
{
        return container_of(attr, struct ib_gid_table_entry, attr)->context;
}
EXPORT_SYMBOL(rdma_read_gid_hw_context);

/**
 * rdma_find_gid - Returns SGID attributes if the matching GID is found.
 * @device: The device to query.
 * @gid: The GID value to search for.
 * @gid_type: The GID type to search for.
 * @ndev: In RoCE, the net device of the device. NULL means ignore.
 *
 * rdma_find_gid() searches for the specified GID value in the software cache.
 *
 * Returns GID attributes if a valid GID is found or returns ERR_PTR for the
 * error. The caller must invoke rdma_put_gid_attr() to release the reference.
 *
 */
const struct ib_gid_attr *rdma_find_gid(struct ib_device *device,
                                        const union ib_gid *gid,
                                        enum ib_gid_type gid_type,
                                        struct net_device *ndev)
{
        unsigned long mask = GID_ATTR_FIND_MASK_GID |
                             GID_ATTR_FIND_MASK_GID_TYPE;
        struct ib_gid_attr gid_attr_val = {.ndev = ndev, .gid_type = gid_type};
        u32 p;

        if (ndev)
                mask |= GID_ATTR_FIND_MASK_NETDEV;

        rdma_for_each_port(device, p) {
                struct ib_gid_table *table;
                unsigned long flags;
                int index;

                table = device->port_data[p].cache.gid;
                read_lock_irqsave(&table->rwlock, flags);
                index = find_gid(table, gid, &gid_attr_val, false, mask, NULL);
                if (index >= 0) {
                        const struct ib_gid_attr *attr;

                        get_gid_entry(table->data_vec[index]);
                        attr = &table->data_vec[index]->attr;
                        read_unlock_irqrestore(&table->rwlock, flags);
                        return attr;
                }
                read_unlock_irqrestore(&table->rwlock, flags);
        }

        return ERR_PTR(-ENOENT);
}
EXPORT_SYMBOL(rdma_find_gid);

int ib_get_cached_pkey(struct ib_device *device,
                       u32               port_num,
                       int               index,
                       u16              *pkey)
{
        struct ib_pkey_cache *cache;
        unsigned long flags;
        int ret = 0;

        if (!rdma_is_port_valid(device, port_num))
                return -EINVAL;

        read_lock_irqsave(&device->cache_lock, flags);

        cache = device->port_data[port_num].cache.pkey;

        if (!cache || index < 0 || index >= cache->table_len)
                ret = -EINVAL;
        else
                *pkey = cache->table[index];

        read_unlock_irqrestore(&device->cache_lock, flags);

        return ret;
}
EXPORT_SYMBOL(ib_get_cached_pkey);

void ib_get_cached_subnet_prefix(struct ib_device *device, u32 port_num,
                                u64 *sn_pfx)
{
        unsigned long flags;

        read_lock_irqsave(&device->cache_lock, flags);
        *sn_pfx = device->port_data[port_num].cache.subnet_prefix;
        read_unlock_irqrestore(&device->cache_lock, flags);
}
EXPORT_SYMBOL(ib_get_cached_subnet_prefix);

int ib_find_cached_pkey(struct ib_device *device, u32 port_num,
                        u16 pkey, u16 *index)
{
        struct ib_pkey_cache *cache;
        unsigned long flags;
        int i;
        int ret = -ENOENT;
        int partial_ix = -1;

        if (!rdma_is_port_valid(device, port_num))
                return -EINVAL;

        read_lock_irqsave(&device->cache_lock, flags);

        cache = device->port_data[port_num].cache.pkey;
        if (!cache) {
                ret = -EINVAL;
                goto err;
        }

        *index = -1;

        for (i = 0; i < cache->table_len; ++i)
                if ((cache->table[i] & 0x7fff) == (pkey & 0x7fff)) {
                        if (cache->table[i] & 0x8000) {
                                *index = i;
                                ret = 0;
                                break;
                        } else {
                                partial_ix = i;
                        }
                }

        if (ret && partial_ix >= 0) {
                *index = partial_ix;
                ret = 0;
        }

err:
        read_unlock_irqrestore(&device->cache_lock, flags);

        return ret;
}
EXPORT_SYMBOL(ib_find_cached_pkey);

int ib_find_exact_cached_pkey(struct ib_device *device, u32 port_num,
                              u16 pkey, u16 *index)
{
        struct ib_pkey_cache *cache;
        unsigned long flags;
        int i;
        int ret = -ENOENT;

        if (!rdma_is_port_valid(device, port_num))
                return -EINVAL;

        read_lock_irqsave(&device->cache_lock, flags);

        cache = device->port_data[port_num].cache.pkey;
        if (!cache) {
                ret = -EINVAL;
                goto err;
        }

        *index = -1;

        for (i = 0; i < cache->table_len; ++i)
                if (cache->table[i] == pkey) {
                        *index = i;
                        ret = 0;
                        break;
                }

err:
        read_unlock_irqrestore(&device->cache_lock, flags);

        return ret;
}
EXPORT_SYMBOL(ib_find_exact_cached_pkey);

int ib_get_cached_lmc(struct ib_device *device, u32 port_num, u8 *lmc)
{
        unsigned long flags;
        int ret = 0;

        if (!rdma_is_port_valid(device, port_num))
                return -EINVAL;

        read_lock_irqsave(&device->cache_lock, flags);
        *lmc = device->port_data[port_num].cache.lmc;
        read_unlock_irqrestore(&device->cache_lock, flags);

        return ret;
}
EXPORT_SYMBOL(ib_get_cached_lmc);

int ib_get_cached_port_state(struct ib_device *device, u32 port_num,
                             enum ib_port_state *port_state)
{
        unsigned long flags;
        int ret = 0;

        if (!rdma_is_port_valid(device, port_num))
                return -EINVAL;

        read_lock_irqsave(&device->cache_lock, flags);
        *port_state = device->port_data[port_num].cache.port_state;
        read_unlock_irqrestore(&device->cache_lock, flags);

        return ret;
}
EXPORT_SYMBOL(ib_get_cached_port_state);

/**
 * rdma_get_gid_attr - Returns GID attributes for a port of a device
 * at a requested gid_index, if a valid GID entry exists.
 * @device:                The device to query.
 * @port_num:                The port number on the device where the GID value
 *                        is to be queried.
 * @index:                Index of the GID table entry whose attributes are to
 *                      be queried.
 *
 * rdma_get_gid_attr() acquires reference count of gid attributes from the
 * cached GID table. Caller must invoke rdma_put_gid_attr() to release
 * reference to gid attribute regardless of link layer.
 *
 * Returns pointer to valid gid attribute or ERR_PTR for the appropriate error
 * code.
 */
const struct ib_gid_attr *
rdma_get_gid_attr(struct ib_device *device, u32 port_num, int index)
{
        const struct ib_gid_attr *attr = ERR_PTR(-ENODATA);
        struct ib_gid_table *table;
        unsigned long flags;

        if (!rdma_is_port_valid(device, port_num))
                return ERR_PTR(-EINVAL);

        table = rdma_gid_table(device, port_num);
        if (index < 0 || index >= table->sz)
                return ERR_PTR(-EINVAL);

        read_lock_irqsave(&table->rwlock, flags);
        if (!is_gid_entry_valid(table->data_vec[index]))
                goto done;

        get_gid_entry(table->data_vec[index]);
        attr = &table->data_vec[index]->attr;
done:
        read_unlock_irqrestore(&table->rwlock, flags);
        return attr;
}
EXPORT_SYMBOL(rdma_get_gid_attr);

/**
 * rdma_query_gid_table - Reads GID table entries of all the ports of a device up to max_entries.
 * @device: The device to query.
 * @entries: Entries where GID entries are returned.
 * @max_entries: Maximum number of entries that can be returned.
 * Entries array must be allocated to hold max_entries number of entries.
 *
 * Returns number of entries on success or appropriate error code.
 */
ssize_t rdma_query_gid_table(struct ib_device *device,
                             struct ib_uverbs_gid_entry *entries,
                             size_t max_entries)
{
        const struct ib_gid_attr *gid_attr;
        ssize_t num_entries = 0, ret;
        struct ib_gid_table *table;
        u32 port_num, i;
        struct net_device *ndev;
        unsigned long flags;

        rdma_for_each_port(device, port_num) {
                table = rdma_gid_table(device, port_num);
                read_lock_irqsave(&table->rwlock, flags);
                for (i = 0; i < table->sz; i++) {
                        if (!is_gid_entry_valid(table->data_vec[i]))
                                continue;
                        if (num_entries >= max_entries) {
                                ret = -EINVAL;
                                goto err;
                        }

                        gid_attr = &table->data_vec[i]->attr;

                        memcpy(&entries->gid, &gid_attr->gid,
                               sizeof(gid_attr->gid));
                        entries->gid_index = gid_attr->index;
                        entries->port_num = gid_attr->port_num;
                        entries->gid_type = gid_attr->gid_type;
                        ndev = rcu_dereference_protected(
                                gid_attr->ndev,
                                lockdep_is_held(&table->rwlock));
                        if (ndev)
                                entries->netdev_ifindex = ndev->ifindex;

                        num_entries++;
                        entries++;
                }
                read_unlock_irqrestore(&table->rwlock, flags);
        }

        return num_entries;
err:
        read_unlock_irqrestore(&table->rwlock, flags);
        return ret;
}
EXPORT_SYMBOL(rdma_query_gid_table);

/**
 * rdma_put_gid_attr - Release reference to the GID attribute
 * @attr:                Pointer to the GID attribute whose reference
 *                        needs to be released.
 *
 * rdma_put_gid_attr() must be used to release reference whose
 * reference is acquired using rdma_get_gid_attr() or any APIs
 * which returns a pointer to the ib_gid_attr regardless of link layer
 * of IB or RoCE.
 *
 */
void rdma_put_gid_attr(const struct ib_gid_attr *attr)
{
        struct ib_gid_table_entry *entry =
                container_of(attr, struct ib_gid_table_entry, attr);

        put_gid_entry(entry);
}
EXPORT_SYMBOL(rdma_put_gid_attr);

/**
 * rdma_hold_gid_attr - Get reference to existing GID attribute
 *
 * @attr:                Pointer to the GID attribute whose reference
 *                        needs to be taken.
 *
 * Increase the reference count to a GID attribute to keep it from being
 * freed. Callers are required to already be holding a reference to attribute.
 *
 */
void rdma_hold_gid_attr(const struct ib_gid_attr *attr)
{
        struct ib_gid_table_entry *entry =
                container_of(attr, struct ib_gid_table_entry, attr);

        get_gid_entry(entry);
}
EXPORT_SYMBOL(rdma_hold_gid_attr);

/**
 * rdma_read_gid_attr_ndev_rcu - Read GID attribute netdevice
 * which must be in UP state.
 *
 * @attr:Pointer to the GID attribute
 *
 * Returns pointer to netdevice if the netdevice was attached to GID and
 * netdevice is in UP state. Caller must hold RCU lock as this API
 * reads the netdev flags which can change while netdevice migrates to
 * different net namespace. Returns ERR_PTR with error code otherwise.
 *
 */
struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr)
{
        struct ib_gid_table_entry *entry =
                        container_of(attr, struct ib_gid_table_entry, attr);
        struct ib_device *device = entry->attr.device;
        struct net_device *ndev = ERR_PTR(-EINVAL);
        u32 port_num = entry->attr.port_num;
        struct ib_gid_table *table;
        unsigned long flags;
        bool valid;

        table = rdma_gid_table(device, port_num);

        read_lock_irqsave(&table->rwlock, flags);
        valid = is_gid_entry_valid(table->data_vec[attr->index]);
        if (valid) {
                ndev = rcu_dereference(attr->ndev);
                if (!ndev)
                        ndev = ERR_PTR(-ENODEV);
        }
        read_unlock_irqrestore(&table->rwlock, flags);
        return ndev;
}
EXPORT_SYMBOL(rdma_read_gid_attr_ndev_rcu);

static int get_lower_dev_vlan(struct net_device *lower_dev,
                              struct netdev_nested_priv *priv)
{
        u16 *vlan_id = (u16 *)priv->data;

        if (is_vlan_dev(lower_dev))
                *vlan_id = vlan_dev_vlan_id(lower_dev);

        /* We are interested only in first level vlan device, so
         * always return 1 to stop iterating over next level devices.
         */
        return 1;
}

/**
 * rdma_read_gid_l2_fields - Read the vlan ID and source MAC address
 *                             of a GID entry.
 *
 * @attr:        GID attribute pointer whose L2 fields to be read
 * @vlan_id:        Pointer to vlan id to fill up if the GID entry has
 *                vlan id. It is optional.
 * @smac:        Pointer to smac to fill up for a GID entry. It is optional.
 *
 * rdma_read_gid_l2_fields() returns 0 on success and returns vlan id
 * (if gid entry has vlan) and source MAC, or returns error.
 */
int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr,
                            u16 *vlan_id, u8 *smac)
{
        struct netdev_nested_priv priv = {
                .data = (void *)vlan_id,
        };
        struct net_device *ndev;

        rcu_read_lock();
        ndev = rcu_dereference(attr->ndev);
        if (!ndev) {
                rcu_read_unlock();
                return -ENODEV;
        }
        if (smac)
                ether_addr_copy(smac, ndev->dev_addr);
        if (vlan_id) {
                *vlan_id = 0xffff;
                if (is_vlan_dev(ndev)) {
                        *vlan_id = vlan_dev_vlan_id(ndev);
                } else {
                        /* If the netdev is upper device and if it's lower
                         * device is vlan device, consider vlan id of
                         * the lower vlan device for this gid entry.
                         */
                        netdev_walk_all_lower_dev_rcu(attr->ndev,
                                        get_lower_dev_vlan, &priv);
                }
        }
        rcu_read_unlock();
        return 0;
}
EXPORT_SYMBOL(rdma_read_gid_l2_fields);

static int config_non_roce_gid_cache(struct ib_device *device,
                                     u32 port, struct ib_port_attr *tprops)
{
        struct ib_gid_attr gid_attr = {};
        struct ib_gid_table *table;
        int ret = 0;
        int i;

        gid_attr.device = device;
        gid_attr.port_num = port;
        table = rdma_gid_table(device, port);

        mutex_lock(&table->lock);
        for (i = 0; i < tprops->gid_tbl_len; ++i) {
                if (!device->ops.query_gid)
                        continue;
                ret = device->ops.query_gid(device, port, i, &gid_attr.gid);
                if (ret) {
                        dev_warn(&device->dev,
                                 "query_gid failed (%d) for index %d\n", ret,
                                 i);
                        goto err;
                }

                if (rdma_protocol_iwarp(device, port)) {
                        struct net_device *ndev;

                        ndev = ib_device_get_netdev(device, port);
                        if (!ndev)
                                continue;
                        RCU_INIT_POINTER(gid_attr.ndev, ndev);
                        dev_put(ndev);
                }

                gid_attr.index = i;
                tprops->subnet_prefix =
                        be64_to_cpu(gid_attr.gid.global.subnet_prefix);
                add_modify_gid(table, &gid_attr);
        }
err:
        mutex_unlock(&table->lock);
        return ret;
}

static int
ib_cache_update(struct ib_device *device, u32 port, bool update_gids,
                bool update_pkeys, bool enforce_security)
{
        struct ib_port_attr       *tprops = NULL;
        struct ib_pkey_cache      *pkey_cache = NULL;
        struct ib_pkey_cache      *old_pkey_cache = NULL;
        int                        i;
        int                        ret;

        if (!rdma_is_port_valid(device, port))
                return -EINVAL;

        tprops = kmalloc(sizeof *tprops, GFP_KERNEL);
        if (!tprops)
                return -ENOMEM;

        ret = ib_query_port(device, port, tprops);
        if (ret) {
                dev_warn(&device->dev, "ib_query_port failed (%d)\n", ret);
                goto err;
        }

        if (!rdma_protocol_roce(device, port) && update_gids) {
                ret = config_non_roce_gid_cache(device, port,
                                                tprops);
                if (ret)
                        goto err;
        }

        update_pkeys &= !!tprops->pkey_tbl_len;

        if (update_pkeys) {
                pkey_cache = kmalloc(struct_size(pkey_cache, table,
                                                 tprops->pkey_tbl_len),
                                     GFP_KERNEL);
                if (!pkey_cache) {
                        ret = -ENOMEM;
                        goto err;
                }

                pkey_cache->table_len = tprops->pkey_tbl_len;

                for (i = 0; i < pkey_cache->table_len; ++i) {
                        ret = ib_query_pkey(device, port, i,
                                            pkey_cache->table + i);
                        if (ret) {
                                dev_warn(&device->dev,
                                         "ib_query_pkey failed (%d) for index %d\n",
                                         ret, i);
                                goto err;
                        }
                }
        }

        write_lock_irq(&device->cache_lock);

        if (update_pkeys) {
                old_pkey_cache = device->port_data[port].cache.pkey;
                device->port_data[port].cache.pkey = pkey_cache;
        }
        device->port_data[port].cache.lmc = tprops->lmc;
        device->port_data[port].cache.port_state = tprops->state;

        device->port_data[port].cache.subnet_prefix = tprops->subnet_prefix;
        write_unlock_irq(&device->cache_lock);

        if (enforce_security)
                ib_security_cache_change(device,
                                         port,
                                         tprops->subnet_prefix);

        kfree(old_pkey_cache);
        kfree(tprops);
        return 0;

err:
        kfree(pkey_cache);
        kfree(tprops);
        return ret;
}

static void ib_cache_event_task(struct work_struct *_work)
{
        struct ib_update_work *work =
                container_of(_work, struct ib_update_work, work);
        int ret;

        /* Before distributing the cache update event, first sync
         * the cache.
         */
        ret = ib_cache_update(work->event.device, work->event.element.port_num,
                              work->event.event == IB_EVENT_GID_CHANGE,
                              work->event.event == IB_EVENT_PKEY_CHANGE,
                              work->enforce_security);

        /* GID event is notified already for individual GID entries by
         * dispatch_gid_change_event(). Hence, notifiy for rest of the
         * events.
         */
        if (!ret && work->event.event != IB_EVENT_GID_CHANGE)
                ib_dispatch_event_clients(&work->event);

        kfree(work);
}

static void ib_generic_event_task(struct work_struct *_work)
{
        struct ib_update_work *work =
                container_of(_work, struct ib_update_work, work);

        ib_dispatch_event_clients(&work->event);
        kfree(work);
}

static bool is_cache_update_event(const struct ib_event *event)
{
        return (event->event == IB_EVENT_PORT_ERR    ||
                event->event == IB_EVENT_PORT_ACTIVE ||
                event->event == IB_EVENT_LID_CHANGE  ||
                event->event == IB_EVENT_PKEY_CHANGE ||
                event->event == IB_EVENT_CLIENT_REREGISTER ||
                event->event == IB_EVENT_GID_CHANGE);
}

/**
 * ib_dispatch_event - Dispatch an asynchronous event
 * @event:Event to dispatch
 *
 * Low-level drivers must call ib_dispatch_event() to dispatch the
 * event to all registered event handlers when an asynchronous event
 * occurs.
 */
void ib_dispatch_event(const struct ib_event *event)
{
        struct ib_update_work *work;

        work = kzalloc(sizeof(*work), GFP_ATOMIC);
        if (!work)
                return;

        if (is_cache_update_event(event))
                INIT_WORK(&work->work, ib_cache_event_task);
        else
                INIT_WORK(&work->work, ib_generic_event_task);

        work->event = *event;
        if (event->event == IB_EVENT_PKEY_CHANGE ||
            event->event == IB_EVENT_GID_CHANGE)
                work->enforce_security = true;

        queue_work(ib_wq, &work->work);
}
EXPORT_SYMBOL(ib_dispatch_event);

int ib_cache_setup_one(struct ib_device *device)
{
        u32 p;
        int err;

        err = gid_table_setup_one(device);
        if (err)
                return err;

        rdma_for_each_port (device, p) {
                err = ib_cache_update(device, p, true, true, true);
                if (err) {
                        gid_table_cleanup_one(device);
                        return err;
                }
        }

        return 0;
}

void ib_cache_release_one(struct ib_device *device)
{
        u32 p;

        /*
         * The release function frees all the cache elements.
         * This function should be called as part of freeing
         * all the device's resources when the cache could no
         * longer be accessed.
         */
        rdma_for_each_port (device, p)
                kfree(device->port_data[p].cache.pkey);

        gid_table_release_one(device);
}

void ib_cache_cleanup_one(struct ib_device *device)
{
        /* The cleanup function waits for all in-progress workqueue
         * elements and cleans up the GID cache. This function should be
         * called after the device was removed from the devices list and
         * all clients were removed, so the cache exists but is
         * non-functional and shouldn't be updated anymore.
         */
        flush_workqueue(ib_wq);
        gid_table_cleanup_one(device);

        /*
         * Flush the wq second time for any pending GID delete work.
         */
        flush_workqueue(ib_wq);
}


















































































































































   11 






    6 






    6 








    6 
    6 
    6 



























    6 









   11 
















    6 


    4 
    4 
    6 







    6 



    3 
    6 









    6 








    1 

























    6 






    4 


    6 
    6 

    6 






    6 

    4 























    1 













    6 




    6 



    6 
    3 

    6 


    6 







    3 


    3 









    1 




    1 




    1 






    1 




    1 




    1 




    3 

    1 
    1 














    6 


    6 

    6 
    1 

    6 






    6 



    6 


    6 





    3 




    6 












    5 



    1 






    1 
    1 
    1 





    1 


    1 


    5 











    1 

















































































































   14 





   11 
   13 
    7 


   11 
    1 


    9 
    8 

    4 
    3 



    6 
    6 
    3 
    6 

    4 
    6 
    1 


    6 
    4 
    4 
    4 
    3 
    3 






    4 


    6 
    1 



    6 
    5 





    6 




    5 


    6 
    6 

















    9 














   14 







   14 



   14 










   14 




    6 

    6 




    6 
    6 

    6 
    6 
    6 

    6 
   14 

    7 






    5 













    3 
    3 





    5 





    5 
























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/sch_sfq.c        Stochastic Fairness Queueing discipline.
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/jiffies.h>
#include <linux/string.h>
#include <linux/in.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/siphash.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/red.h>


/*        Stochastic Fairness Queuing algorithm.
        =======================================

        Source:
        Paul E. McKenney "Stochastic Fairness Queuing",
        IEEE INFOCOMM'90 Proceedings, San Francisco, 1990.

        Paul E. McKenney "Stochastic Fairness Queuing",
        "Interworking: Research and Experience", v.2, 1991, p.113-131.


        See also:
        M. Shreedhar and George Varghese "Efficient Fair
        Queuing using Deficit Round Robin", Proc. SIGCOMM 95.


        This is not the thing that is usually called (W)FQ nowadays.
        It does not use any timestamp mechanism, but instead
        processes queues in round-robin order.

        ADVANTAGE:

        - It is very cheap. Both CPU and memory requirements are minimal.

        DRAWBACKS:

        - "Stochastic" -> It is not 100% fair.
        When hash collisions occur, several flows are considered as one.

        - "Round-robin" -> It introduces larger delays than virtual clock
        based schemes, and should not be used for isolating interactive
        traffic        from non-interactive. It means, that this scheduler
        should be used as leaf of CBQ or P3, which put interactive traffic
        to higher priority band.

        We still need true WFQ for top level CSZ, but using WFQ
        for the best effort traffic is absolutely pointless:
        SFQ is superior for this purpose.

        IMPLEMENTATION:
        This implementation limits :
        - maximal queue length per flow to 127 packets.
        - max mtu to 2^18-1;
        - max 65408 flows,
        - number of hash buckets to 65536.

        It is easy to increase these values, but not in flight.  */

#define SFQ_MAX_DEPTH                127 /* max number of packets per flow */
#define SFQ_DEFAULT_FLOWS        128
#define SFQ_MAX_FLOWS                (0x10000 - SFQ_MAX_DEPTH - 1) /* max number of flows */
#define SFQ_EMPTY_SLOT                0xffff
#define SFQ_DEFAULT_HASH_DIVISOR 1024

/* This type should contain at least SFQ_MAX_DEPTH + 1 + SFQ_MAX_FLOWS values */
typedef u16 sfq_index;

/*
 * We dont use pointers to save space.
 * Small indexes [0 ... SFQ_MAX_FLOWS - 1] are 'pointers' to slots[] array
 * while following values [SFQ_MAX_FLOWS ... SFQ_MAX_FLOWS + SFQ_MAX_DEPTH]
 * are 'pointers' to dep[] array
 */
struct sfq_head {
        sfq_index        next;
        sfq_index        prev;
};

struct sfq_slot {
        struct sk_buff        *skblist_next;
        struct sk_buff        *skblist_prev;
        sfq_index        qlen; /* number of skbs in skblist */
        sfq_index        next; /* next slot in sfq RR chain */
        struct sfq_head dep; /* anchor in dep[] chains */
        unsigned short        hash; /* hash value (index in ht[]) */
        int                allot; /* credit for this slot */

        unsigned int    backlog;
        struct red_vars vars;
};

struct sfq_sched_data {
/* frequently used fields */
        int                limit;                /* limit of total number of packets in this qdisc */
        unsigned int        divisor;        /* number of slots in hash table */
        u8                headdrop;
        u8                maxdepth;        /* limit of packets per flow */

        siphash_key_t         perturbation;
        u8                cur_depth;        /* depth of longest slot */
        u8                flags;
        struct tcf_proto __rcu *filter_list;
        struct tcf_block *block;
        sfq_index        *ht;                /* Hash table ('divisor' slots) */
        struct sfq_slot        *slots;                /* Flows table ('maxflows' entries) */

        struct red_parms *red_parms;
        struct tc_sfqred_stats stats;
        struct sfq_slot *tail;                /* current slot in round */

        struct sfq_head        dep[SFQ_MAX_DEPTH + 1];
                                        /* Linked lists of slots, indexed by depth
                                         * dep[0] : list of unused flows
                                         * dep[1] : list of flows with 1 packet
                                         * dep[X] : list of flows with X packets
                                         */

        unsigned int        maxflows;        /* number of flows in flows array */
        int                perturb_period;
        unsigned int        quantum;        /* Allotment per round: MUST BE >= MTU */
        struct timer_list perturb_timer;
        struct Qdisc        *sch;
};

/*
 * sfq_head are either in a sfq_slot or in dep[] array
 */
static inline struct sfq_head *sfq_dep_head(struct sfq_sched_data *q, sfq_index val)
{
        if (val < SFQ_MAX_FLOWS)
                return &q->slots[val].dep;
        return &q->dep[val - SFQ_MAX_FLOWS];
}

static unsigned int sfq_hash(const struct sfq_sched_data *q,
                             const struct sk_buff *skb)
{
        return skb_get_hash_perturb(skb, &q->perturbation) & (q->divisor - 1);
}

static unsigned int sfq_classify(struct sk_buff *skb, struct Qdisc *sch,
                                 int *qerr)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        struct tcf_result res;
        struct tcf_proto *fl;
        int result;

        if (TC_H_MAJ(skb->priority) == sch->handle &&
            TC_H_MIN(skb->priority) > 0 &&
            TC_H_MIN(skb->priority) <= q->divisor)
                return TC_H_MIN(skb->priority);

        fl = rcu_dereference_bh(q->filter_list);
        if (!fl)
                return sfq_hash(q, skb) + 1;

        *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
        result = tcf_classify(skb, NULL, fl, &res, false);
        if (result >= 0) {
#ifdef CONFIG_NET_CLS_ACT
                switch (result) {
                case TC_ACT_STOLEN:
                case TC_ACT_QUEUED:
                case TC_ACT_TRAP:
                        *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
                        fallthrough;
                case TC_ACT_SHOT:
                        return 0;
                }
#endif
                if (TC_H_MIN(res.classid) <= q->divisor)
                        return TC_H_MIN(res.classid);
        }
        return 0;
}

/*
 * x : slot number [0 .. SFQ_MAX_FLOWS - 1]
 */
static inline void sfq_link(struct sfq_sched_data *q, sfq_index x)
{
        sfq_index p, n;
        struct sfq_slot *slot = &q->slots[x];
        int qlen = slot->qlen;

        p = qlen + SFQ_MAX_FLOWS;
        n = q->dep[qlen].next;

        slot->dep.next = n;
        slot->dep.prev = p;

        q->dep[qlen].next = x;                /* sfq_dep_head(q, p)->next = x */
        sfq_dep_head(q, n)->prev = x;
}

#define sfq_unlink(q, x, n, p)                        \
        do {                                        \
                n = q->slots[x].dep.next;        \
                p = q->slots[x].dep.prev;        \
                sfq_dep_head(q, p)->next = n;        \
                sfq_dep_head(q, n)->prev = p;        \
        } while (0)


static inline void sfq_dec(struct sfq_sched_data *q, sfq_index x)
{
        sfq_index p, n;
        int d;

        sfq_unlink(q, x, n, p);

        d = q->slots[x].qlen--;
        if (n == p && q->cur_depth == d)
                q->cur_depth--;
        sfq_link(q, x);
}

static inline void sfq_inc(struct sfq_sched_data *q, sfq_index x)
{
        sfq_index p, n;
        int d;

        sfq_unlink(q, x, n, p);

        d = ++q->slots[x].qlen;
        if (q->cur_depth < d)
                q->cur_depth = d;
        sfq_link(q, x);
}

/* helper functions : might be changed when/if skb use a standard list_head */

/* remove one skb from tail of slot queue */
static inline struct sk_buff *slot_dequeue_tail(struct sfq_slot *slot)
{
        struct sk_buff *skb = slot->skblist_prev;

        slot->skblist_prev = skb->prev;
        skb->prev->next = (struct sk_buff *)slot;
        skb->next = skb->prev = NULL;
        return skb;
}

/* remove one skb from head of slot queue */
static inline struct sk_buff *slot_dequeue_head(struct sfq_slot *slot)
{
        struct sk_buff *skb = slot->skblist_next;

        slot->skblist_next = skb->next;
        skb->next->prev = (struct sk_buff *)slot;
        skb->next = skb->prev = NULL;
        return skb;
}

static inline void slot_queue_init(struct sfq_slot *slot)
{
        memset(slot, 0, sizeof(*slot));
        slot->skblist_prev = slot->skblist_next = (struct sk_buff *)slot;
}

/* add skb to slot queue (tail add) */
static inline void slot_queue_add(struct sfq_slot *slot, struct sk_buff *skb)
{
        skb->prev = slot->skblist_prev;
        skb->next = (struct sk_buff *)slot;
        slot->skblist_prev->next = skb;
        slot->skblist_prev = skb;
}

static unsigned int sfq_drop(struct Qdisc *sch, struct sk_buff **to_free)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        sfq_index x, d = q->cur_depth;
        struct sk_buff *skb;
        unsigned int len;
        struct sfq_slot *slot;

        /* Queue is full! Find the longest slot and drop tail packet from it */
        if (d > 1) {
                x = q->dep[d].next;
                slot = &q->slots[x];
drop:
                skb = q->headdrop ? slot_dequeue_head(slot) : slot_dequeue_tail(slot);
                len = qdisc_pkt_len(skb);
                slot->backlog -= len;
                sfq_dec(q, x);
                sch->q.qlen--;
                qdisc_qstats_backlog_dec(sch, skb);
                qdisc_drop(skb, sch, to_free);
                return len;
        }

        if (d == 1) {
                /* It is difficult to believe, but ALL THE SLOTS HAVE LENGTH 1. */
                x = q->tail->next;
                slot = &q->slots[x];
                q->tail->next = slot->next;
                q->ht[slot->hash] = SFQ_EMPTY_SLOT;
                goto drop;
        }

        return 0;
}

/* Is ECN parameter configured */
static int sfq_prob_mark(const struct sfq_sched_data *q)
{
        return q->flags & TC_RED_ECN;
}

/* Should packets over max threshold just be marked */
static int sfq_hard_mark(const struct sfq_sched_data *q)
{
        return (q->flags & (TC_RED_ECN | TC_RED_HARDDROP)) == TC_RED_ECN;
}

static int sfq_headdrop(const struct sfq_sched_data *q)
{
        return q->headdrop;
}

static int
sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        unsigned int hash, dropped;
        sfq_index x, qlen;
        struct sfq_slot *slot;
        int ret;
        struct sk_buff *head;
        int delta;

        hash = sfq_classify(skb, sch, &ret);
        if (hash == 0) {
                if (ret & __NET_XMIT_BYPASS)
                        qdisc_qstats_drop(sch);
                __qdisc_drop(skb, to_free);
                return ret;
        }
        hash--;

        x = q->ht[hash];
        slot = &q->slots[x];
        if (x == SFQ_EMPTY_SLOT) {
                x = q->dep[0].next; /* get a free slot */
                if (x >= SFQ_MAX_FLOWS)
                        return qdisc_drop(skb, sch, to_free);
                q->ht[hash] = x;
                slot = &q->slots[x];
                slot->hash = hash;
                slot->backlog = 0; /* should already be 0 anyway... */
                red_set_vars(&slot->vars);
                goto enqueue;
        }
        if (q->red_parms) {
                slot->vars.qavg = red_calc_qavg_no_idle_time(q->red_parms,
                                                        &slot->vars,
                                                        slot->backlog);
                switch (red_action(q->red_parms,
                                   &slot->vars,
                                   slot->vars.qavg)) {
                case RED_DONT_MARK:
                        break;

                case RED_PROB_MARK:
                        qdisc_qstats_overlimit(sch);
                        if (sfq_prob_mark(q)) {
                                /* We know we have at least one packet in queue */
                                if (sfq_headdrop(q) &&
                                    INET_ECN_set_ce(slot->skblist_next)) {
                                        q->stats.prob_mark_head++;
                                        break;
                                }
                                if (INET_ECN_set_ce(skb)) {
                                        q->stats.prob_mark++;
                                        break;
                                }
                        }
                        q->stats.prob_drop++;
                        goto congestion_drop;

                case RED_HARD_MARK:
                        qdisc_qstats_overlimit(sch);
                        if (sfq_hard_mark(q)) {
                                /* We know we have at least one packet in queue */
                                if (sfq_headdrop(q) &&
                                    INET_ECN_set_ce(slot->skblist_next)) {
                                        q->stats.forced_mark_head++;
                                        break;
                                }
                                if (INET_ECN_set_ce(skb)) {
                                        q->stats.forced_mark++;
                                        break;
                                }
                        }
                        q->stats.forced_drop++;
                        goto congestion_drop;
                }
        }

        if (slot->qlen >= q->maxdepth) {
congestion_drop:
                if (!sfq_headdrop(q))
                        return qdisc_drop(skb, sch, to_free);

                /* We know we have at least one packet in queue */
                head = slot_dequeue_head(slot);
                delta = qdisc_pkt_len(head) - qdisc_pkt_len(skb);
                sch->qstats.backlog -= delta;
                slot->backlog -= delta;
                qdisc_drop(head, sch, to_free);

                slot_queue_add(slot, skb);
                qdisc_tree_reduce_backlog(sch, 0, delta);
                return NET_XMIT_CN;
        }

enqueue:
        qdisc_qstats_backlog_inc(sch, skb);
        slot->backlog += qdisc_pkt_len(skb);
        slot_queue_add(slot, skb);
        sfq_inc(q, x);
        if (slot->qlen == 1) {                /* The flow is new */
                if (q->tail == NULL) {        /* It is the first flow */
                        slot->next = x;
                } else {
                        slot->next = q->tail->next;
                        q->tail->next = x;
                }
                /* We put this flow at the end of our flow list.
                 * This might sound unfair for a new flow to wait after old ones,
                 * but we could endup servicing new flows only, and freeze old ones.
                 */
                q->tail = slot;
                /* We could use a bigger initial quantum for new flows */
                slot->allot = q->quantum;
        }
        if (++sch->q.qlen <= q->limit)
                return NET_XMIT_SUCCESS;

        qlen = slot->qlen;
        dropped = sfq_drop(sch, to_free);
        /* Return Congestion Notification only if we dropped a packet
         * from this flow.
         */
        if (qlen != slot->qlen) {
                qdisc_tree_reduce_backlog(sch, 0, dropped - qdisc_pkt_len(skb));
                return NET_XMIT_CN;
        }

        /* As we dropped a packet, better let upper stack know this */
        qdisc_tree_reduce_backlog(sch, 1, dropped);
        return NET_XMIT_SUCCESS;
}

static struct sk_buff *
sfq_dequeue(struct Qdisc *sch)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        struct sk_buff *skb;
        sfq_index a, next_a;
        struct sfq_slot *slot;

        /* No active slots */
        if (q->tail == NULL)
                return NULL;

next_slot:
        a = q->tail->next;
        slot = &q->slots[a];
        if (slot->allot <= 0) {
                q->tail = slot;
                slot->allot += q->quantum;
                goto next_slot;
        }
        skb = slot_dequeue_head(slot);
        sfq_dec(q, a);
        qdisc_bstats_update(sch, skb);
        sch->q.qlen--;
        qdisc_qstats_backlog_dec(sch, skb);
        slot->backlog -= qdisc_pkt_len(skb);
        /* Is the slot empty? */
        if (slot->qlen == 0) {
                q->ht[slot->hash] = SFQ_EMPTY_SLOT;
                next_a = slot->next;
                if (a == next_a) {
                        q->tail = NULL; /* no more active slots */
                        return skb;
                }
                q->tail->next = next_a;
        } else {
                slot->allot -= qdisc_pkt_len(skb);
        }
        return skb;
}

static void
sfq_reset(struct Qdisc *sch)
{
        struct sk_buff *skb;

        while ((skb = sfq_dequeue(sch)) != NULL)
                rtnl_kfree_skbs(skb, skb);
}

/*
 * When q->perturbation is changed, we rehash all queued skbs
 * to avoid OOO (Out Of Order) effects.
 * We dont use sfq_dequeue()/sfq_enqueue() because we dont want to change
 * counters.
 */
static void sfq_rehash(struct Qdisc *sch)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        struct sk_buff *skb;
        int i;
        struct sfq_slot *slot;
        struct sk_buff_head list;
        int dropped = 0;
        unsigned int drop_len = 0;

        __skb_queue_head_init(&list);

        for (i = 0; i < q->maxflows; i++) {
                slot = &q->slots[i];
                if (!slot->qlen)
                        continue;
                while (slot->qlen) {
                        skb = slot_dequeue_head(slot);
                        sfq_dec(q, i);
                        __skb_queue_tail(&list, skb);
                }
                slot->backlog = 0;
                red_set_vars(&slot->vars);
                q->ht[slot->hash] = SFQ_EMPTY_SLOT;
        }
        q->tail = NULL;

        while ((skb = __skb_dequeue(&list)) != NULL) {
                unsigned int hash = sfq_hash(q, skb);
                sfq_index x = q->ht[hash];

                slot = &q->slots[x];
                if (x == SFQ_EMPTY_SLOT) {
                        x = q->dep[0].next; /* get a free slot */
                        if (x >= SFQ_MAX_FLOWS) {
drop:
                                qdisc_qstats_backlog_dec(sch, skb);
                                drop_len += qdisc_pkt_len(skb);
                                kfree_skb(skb);
                                dropped++;
                                continue;
                        }
                        q->ht[hash] = x;
                        slot = &q->slots[x];
                        slot->hash = hash;
                }
                if (slot->qlen >= q->maxdepth)
                        goto drop;
                slot_queue_add(slot, skb);
                if (q->red_parms)
                        slot->vars.qavg = red_calc_qavg(q->red_parms,
                                                        &slot->vars,
                                                        slot->backlog);
                slot->backlog += qdisc_pkt_len(skb);
                sfq_inc(q, x);
                if (slot->qlen == 1) {                /* The flow is new */
                        if (q->tail == NULL) {        /* It is the first flow */
                                slot->next = x;
                        } else {
                                slot->next = q->tail->next;
                                q->tail->next = x;
                        }
                        q->tail = slot;
                        slot->allot = q->quantum;
                }
        }
        sch->q.qlen -= dropped;
        qdisc_tree_reduce_backlog(sch, dropped, drop_len);
}

static void sfq_perturbation(struct timer_list *t)
{
        struct sfq_sched_data *q = from_timer(q, t, perturb_timer);
        struct Qdisc *sch = q->sch;
        spinlock_t *root_lock;
        siphash_key_t nkey;
        int period;

        get_random_bytes(&nkey, sizeof(nkey));
        rcu_read_lock();
        root_lock = qdisc_lock(qdisc_root_sleeping(sch));
        spin_lock(root_lock);
        q->perturbation = nkey;
        if (!q->filter_list && q->tail)
                sfq_rehash(sch);
        spin_unlock(root_lock);

        /* q->perturb_period can change under us from
         * sfq_change() and sfq_destroy().
         */
        period = READ_ONCE(q->perturb_period);
        if (period)
                mod_timer(&q->perturb_timer, jiffies + period);
        rcu_read_unlock();
}

static int sfq_change(struct Qdisc *sch, struct nlattr *opt,
                      struct netlink_ext_ack *extack)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        struct tc_sfq_qopt *ctl = nla_data(opt);
        struct tc_sfq_qopt_v1 *ctl_v1 = NULL;
        unsigned int qlen, dropped = 0;
        struct red_parms *p = NULL;
        struct sk_buff *to_free = NULL;
        struct sk_buff *tail = NULL;

        if (opt->nla_len < nla_attr_size(sizeof(*ctl)))
                return -EINVAL;
        if (opt->nla_len >= nla_attr_size(sizeof(*ctl_v1)))
                ctl_v1 = nla_data(opt);
        if (ctl->divisor &&
            (!is_power_of_2(ctl->divisor) || ctl->divisor > 65536))
                return -EINVAL;

        if ((int)ctl->quantum < 0) {
                NL_SET_ERR_MSG_MOD(extack, "invalid quantum");
                return -EINVAL;
        }
        if (ctl_v1 && !red_check_params(ctl_v1->qth_min, ctl_v1->qth_max,
                                        ctl_v1->Wlog, ctl_v1->Scell_log, NULL))
                return -EINVAL;
        if (ctl_v1 && ctl_v1->qth_min) {
                p = kmalloc(sizeof(*p), GFP_KERNEL);
                if (!p)
                        return -ENOMEM;
        }
        sch_tree_lock(sch);
        if (ctl->quantum)
                q->quantum = ctl->quantum;
        WRITE_ONCE(q->perturb_period, ctl->perturb_period * HZ);
        if (ctl->flows)
                q->maxflows = min_t(u32, ctl->flows, SFQ_MAX_FLOWS);
        if (ctl->divisor) {
                q->divisor = ctl->divisor;
                q->maxflows = min_t(u32, q->maxflows, q->divisor);
        }
        if (ctl_v1) {
                if (ctl_v1->depth)
                        q->maxdepth = min_t(u32, ctl_v1->depth, SFQ_MAX_DEPTH);
                if (p) {
                        swap(q->red_parms, p);
                        red_set_parms(q->red_parms,
                                      ctl_v1->qth_min, ctl_v1->qth_max,
                                      ctl_v1->Wlog,
                                      ctl_v1->Plog, ctl_v1->Scell_log,
                                      NULL,
                                      ctl_v1->max_P);
                }
                q->flags = ctl_v1->flags;
                q->headdrop = ctl_v1->headdrop;
        }
        if (ctl->limit) {
                q->limit = min_t(u32, ctl->limit, q->maxdepth * q->maxflows);
                q->maxflows = min_t(u32, q->maxflows, q->limit);
        }

        qlen = sch->q.qlen;
        while (sch->q.qlen > q->limit) {
                dropped += sfq_drop(sch, &to_free);
                if (!tail)
                        tail = to_free;
        }

        rtnl_kfree_skbs(to_free, tail);
        qdisc_tree_reduce_backlog(sch, qlen - sch->q.qlen, dropped);

        del_timer(&q->perturb_timer);
        if (q->perturb_period) {
                mod_timer(&q->perturb_timer, jiffies + q->perturb_period);
                get_random_bytes(&q->perturbation, sizeof(q->perturbation));
        }
        sch_tree_unlock(sch);
        kfree(p);
        return 0;
}

static void *sfq_alloc(size_t sz)
{
        return  kvmalloc(sz, GFP_KERNEL);
}

static void sfq_free(void *addr)
{
        kvfree(addr);
}

static void sfq_destroy(struct Qdisc *sch)
{
        struct sfq_sched_data *q = qdisc_priv(sch);

        tcf_block_put(q->block);
        WRITE_ONCE(q->perturb_period, 0);
        del_timer_sync(&q->perturb_timer);
        sfq_free(q->ht);
        sfq_free(q->slots);
        kfree(q->red_parms);
}

static int sfq_init(struct Qdisc *sch, struct nlattr *opt,
                    struct netlink_ext_ack *extack)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        int i;
        int err;

        q->sch = sch;
        timer_setup(&q->perturb_timer, sfq_perturbation, TIMER_DEFERRABLE);

        err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
        if (err)
                return err;

        for (i = 0; i < SFQ_MAX_DEPTH + 1; i++) {
                q->dep[i].next = i + SFQ_MAX_FLOWS;
                q->dep[i].prev = i + SFQ_MAX_FLOWS;
        }

        q->limit = SFQ_MAX_DEPTH;
        q->maxdepth = SFQ_MAX_DEPTH;
        q->cur_depth = 0;
        q->tail = NULL;
        q->divisor = SFQ_DEFAULT_HASH_DIVISOR;
        q->maxflows = SFQ_DEFAULT_FLOWS;
        q->quantum = psched_mtu(qdisc_dev(sch));
        q->perturb_period = 0;
        get_random_bytes(&q->perturbation, sizeof(q->perturbation));

        if (opt) {
                int err = sfq_change(sch, opt, extack);
                if (err)
                        return err;
        }

        q->ht = sfq_alloc(sizeof(q->ht[0]) * q->divisor);
        q->slots = sfq_alloc(sizeof(q->slots[0]) * q->maxflows);
        if (!q->ht || !q->slots) {
                /* Note: sfq_destroy() will be called by our caller */
                return -ENOMEM;
        }

        for (i = 0; i < q->divisor; i++)
                q->ht[i] = SFQ_EMPTY_SLOT;

        for (i = 0; i < q->maxflows; i++) {
                slot_queue_init(&q->slots[i]);
                sfq_link(q, i);
        }
        if (q->limit >= 1)
                sch->flags |= TCQ_F_CAN_BYPASS;
        else
                sch->flags &= ~TCQ_F_CAN_BYPASS;
        return 0;
}

static int sfq_dump(struct Qdisc *sch, struct sk_buff *skb)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        unsigned char *b = skb_tail_pointer(skb);
        struct tc_sfq_qopt_v1 opt;
        struct red_parms *p = q->red_parms;

        memset(&opt, 0, sizeof(opt));
        opt.v0.quantum        = q->quantum;
        opt.v0.perturb_period = q->perturb_period / HZ;
        opt.v0.limit        = q->limit;
        opt.v0.divisor        = q->divisor;
        opt.v0.flows        = q->maxflows;
        opt.depth        = q->maxdepth;
        opt.headdrop        = q->headdrop;

        if (p) {
                opt.qth_min        = p->qth_min >> p->Wlog;
                opt.qth_max        = p->qth_max >> p->Wlog;
                opt.Wlog        = p->Wlog;
                opt.Plog        = p->Plog;
                opt.Scell_log        = p->Scell_log;
                opt.max_P        = p->max_P;
        }
        memcpy(&opt.stats, &q->stats, sizeof(opt.stats));
        opt.flags        = q->flags;

        if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
                goto nla_put_failure;

        return skb->len;

nla_put_failure:
        nlmsg_trim(skb, b);
        return -1;
}

static struct Qdisc *sfq_leaf(struct Qdisc *sch, unsigned long arg)
{
        return NULL;
}

static unsigned long sfq_find(struct Qdisc *sch, u32 classid)
{
        return 0;
}

static unsigned long sfq_bind(struct Qdisc *sch, unsigned long parent,
                              u32 classid)
{
        return 0;
}

static void sfq_unbind(struct Qdisc *q, unsigned long cl)
{
}

static struct tcf_block *sfq_tcf_block(struct Qdisc *sch, unsigned long cl,
                                       struct netlink_ext_ack *extack)
{
        struct sfq_sched_data *q = qdisc_priv(sch);

        if (cl)
                return NULL;
        return q->block;
}

static int sfq_dump_class(struct Qdisc *sch, unsigned long cl,
                          struct sk_buff *skb, struct tcmsg *tcm)
{
        tcm->tcm_handle |= TC_H_MIN(cl);
        return 0;
}

static int sfq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
                                struct gnet_dump *d)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        sfq_index idx = q->ht[cl - 1];
        struct gnet_stats_queue qs = { 0 };
        struct tc_sfq_xstats xstats = { 0 };

        if (idx != SFQ_EMPTY_SLOT) {
                const struct sfq_slot *slot = &q->slots[idx];

                xstats.allot = slot->allot;
                qs.qlen = slot->qlen;
                qs.backlog = slot->backlog;
        }
        if (gnet_stats_copy_queue(d, NULL, &qs, qs.qlen) < 0)
                return -1;
        return gnet_stats_copy_app(d, &xstats, sizeof(xstats));
}

static void sfq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
{
        struct sfq_sched_data *q = qdisc_priv(sch);
        unsigned int i;

        if (arg->stop)
                return;

        for (i = 0; i < q->divisor; i++) {
                if (q->ht[i] == SFQ_EMPTY_SLOT) {
                        arg->count++;
                        continue;
                }
                if (!tc_qdisc_stats_dump(sch, i + 1, arg))
                        break;
        }
}

static const struct Qdisc_class_ops sfq_class_ops = {
        .leaf                =        sfq_leaf,
        .find                =        sfq_find,
        .tcf_block        =        sfq_tcf_block,
        .bind_tcf        =        sfq_bind,
        .unbind_tcf        =        sfq_unbind,
        .dump                =        sfq_dump_class,
        .dump_stats        =        sfq_dump_class_stats,
        .walk                =        sfq_walk,
};

static struct Qdisc_ops sfq_qdisc_ops __read_mostly = {
        .cl_ops                =        &sfq_class_ops,
        .id                =        "sfq",
        .priv_size        =        sizeof(struct sfq_sched_data),
        .enqueue        =        sfq_enqueue,
        .dequeue        =        sfq_dequeue,
        .peek                =        qdisc_peek_dequeued,
        .init                =        sfq_init,
        .reset                =        sfq_reset,
        .destroy        =        sfq_destroy,
        .change                =        NULL,
        .dump                =        sfq_dump,
        .owner                =        THIS_MODULE,
};
MODULE_ALIAS_NET_SCH("sfq");

static int __init sfq_module_init(void)
{
        return register_qdisc(&sfq_qdisc_ops);
}
static void __exit sfq_module_exit(void)
{
        unregister_qdisc(&sfq_qdisc_ops);
}
module_init(sfq_module_init)
module_exit(sfq_module_exit)
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Stochastic Fairness qdisc");
























































    1 













    1 







    1 





    1 




















    1 

    1 








    1 


    1 


























    1 









































































    1 




    1 




    1 




    1 




    1 




    1 




    1 






































































































































    1 



    1 



    1 












    1 








































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
// SPDX-License-Identifier: GPL-2.0-only
 /*
 * USB Driver for ALi m5602 based webcams
 *
 * Copyright (C) 2008 Erik Andrén
 * Copyright (C) 2007 Ilyes Gouta. Based on the m5603x Linux Driver Project.
 * Copyright (C) 2005 m5603x Linux Driver Project <m5602@x3ng.com.br>
 *
 * Portions of code to USB interface and ALi driver software,
 * Copyright (c) 2006 Willem Duinker
 * v4l2 interface modeled after the V4L2 driver
 * for SN9C10x PC Camera Controllers
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include "m5602_ov9650.h"
#include "m5602_ov7660.h"
#include "m5602_mt9m111.h"
#include "m5602_po1030.h"
#include "m5602_s5k83a.h"
#include "m5602_s5k4aa.h"

/* Kernel module parameters */
int force_sensor;
static bool dump_bridge;
bool dump_sensor;

static const struct usb_device_id m5602_table[] = {
        {USB_DEVICE(0x0402, 0x5602)},
        {}
};

MODULE_DEVICE_TABLE(usb, m5602_table);

/* A skeleton used for sending messages to the sensor */
static const unsigned char sensor_urb_skeleton[] = {
        0x23, M5602_XB_GPIO_EN_H, 0x81, 0x06,
        0x23, M5602_XB_MISC_CTRL, 0x81, 0x80,
        0x13, M5602_XB_I2C_DEV_ADDR, 0x81, 0x00,
        0x13, M5602_XB_I2C_REG_ADDR, 0x81, 0x00,
        0x13, M5602_XB_I2C_DATA, 0x81, 0x00,
        0x13, M5602_XB_I2C_CTRL, 0x81, 0x11
};

/* A skeleton used for sending messages to the m5602 bridge */
static const unsigned char bridge_urb_skeleton[] = {
        0x13, 0x00, 0x81, 0x00
};

/* Reads a byte from the m5602 */
int m5602_read_bridge(struct sd *sd, const u8 address, u8 *i2c_data)
{
        int err;
        struct gspca_dev *gspca_dev = (struct gspca_dev *) sd;
        struct usb_device *udev = sd->gspca_dev.dev;
        __u8 *buf = sd->gspca_dev.usb_buf;

        err = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
                              0x04, 0xc0, 0x14,
                              0x8100 + address, buf,
                              1, M5602_URB_MSG_TIMEOUT);
        *i2c_data = buf[0];

        gspca_dbg(gspca_dev, D_CONF, "Reading bridge register 0x%x containing 0x%x\n",
                  address, *i2c_data);

        /* usb_control_msg(...) returns the number of bytes sent upon success,
        mask that and return zero instead*/
        return (err < 0) ? err : 0;
}

/* Writes a byte to the m5602 */
int m5602_write_bridge(struct sd *sd, const u8 address, const u8 i2c_data)
{
        int err;
        struct gspca_dev *gspca_dev = (struct gspca_dev *) sd;
        struct usb_device *udev = sd->gspca_dev.dev;
        __u8 *buf = sd->gspca_dev.usb_buf;

        gspca_dbg(gspca_dev, D_CONF, "Writing bridge register 0x%x with 0x%x\n",
                  address, i2c_data);

        memcpy(buf, bridge_urb_skeleton,
               sizeof(bridge_urb_skeleton));
        buf[1] = address;
        buf[3] = i2c_data;

        err = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
                                0x04, 0x40, 0x19,
                                0x0000, buf,
                                4, M5602_URB_MSG_TIMEOUT);

        /* usb_control_msg(...) returns the number of bytes sent upon success,
           mask that and return zero instead */
        return (err < 0) ? err : 0;
}

static int m5602_wait_for_i2c(struct sd *sd)
{
        int err;
        u8 data;

        do {
                err = m5602_read_bridge(sd, M5602_XB_I2C_STATUS, &data);
        } while ((data & I2C_BUSY) && !err);
        return err;
}

int m5602_read_sensor(struct sd *sd, const u8 address,
                       u8 *i2c_data, const u8 len)
{
        int err, i;
        struct gspca_dev *gspca_dev = (struct gspca_dev *) sd;

        if (!len || len > sd->sensor->i2c_regW)
                return -EINVAL;

        err = m5602_wait_for_i2c(sd);
        if (err < 0)
                return err;

        err = m5602_write_bridge(sd, M5602_XB_I2C_DEV_ADDR,
                                 sd->sensor->i2c_slave_id);
        if (err < 0)
                return err;

        err = m5602_write_bridge(sd, M5602_XB_I2C_REG_ADDR, address);
        if (err < 0)
                return err;

        /* Sensors with registers that are of only
           one byte width are differently read */

        /* FIXME: This works with the ov9650, but has issues with the po1030 */
        if (sd->sensor->i2c_regW == 1) {
                err = m5602_write_bridge(sd, M5602_XB_I2C_CTRL, 1);
                if (err < 0)
                        return err;

                err = m5602_write_bridge(sd, M5602_XB_I2C_CTRL, 0x08);
        } else {
                err = m5602_write_bridge(sd, M5602_XB_I2C_CTRL, 0x18 + len);
        }

        for (i = 0; (i < len) && !err; i++) {
                err = m5602_wait_for_i2c(sd);
                if (err < 0)
                        return err;

                err = m5602_read_bridge(sd, M5602_XB_I2C_DATA, &(i2c_data[i]));

                gspca_dbg(gspca_dev, D_CONF, "Reading sensor register 0x%x containing 0x%x\n",
                          address, *i2c_data);
        }
        return err;
}

int m5602_write_sensor(struct sd *sd, const u8 address,
                        u8 *i2c_data, const u8 len)
{
        int err, i;
        u8 *p;
        struct gspca_dev *gspca_dev = (struct gspca_dev *) sd;
        struct usb_device *udev = sd->gspca_dev.dev;
        __u8 *buf = sd->gspca_dev.usb_buf;

        /* No sensor with a data width larger than 16 bits has yet been seen */
        if (len > sd->sensor->i2c_regW || !len)
                return -EINVAL;

        memcpy(buf, sensor_urb_skeleton,
               sizeof(sensor_urb_skeleton));

        buf[11] = sd->sensor->i2c_slave_id;
        buf[15] = address;

        /* Special case larger sensor writes */
        p = buf + 16;

        /* Copy a four byte write sequence for each byte to be written to */
        for (i = 0; i < len; i++) {
                memcpy(p, sensor_urb_skeleton + 16, 4);
                p[3] = i2c_data[i];
                p += 4;
                gspca_dbg(gspca_dev, D_CONF, "Writing sensor register 0x%x with 0x%x\n",
                          address, i2c_data[i]);
        }

        /* Copy the tailer */
        memcpy(p, sensor_urb_skeleton + 20, 4);

        /* Set the total length */
        p[3] = 0x10 + len;

        err = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
                              0x04, 0x40, 0x19,
                              0x0000, buf,
                              20 + len * 4, M5602_URB_MSG_TIMEOUT);

        return (err < 0) ? err : 0;
}

/* Dump all the registers of the m5602 bridge,
   unfortunately this breaks the camera until it's power cycled */
static void m5602_dump_bridge(struct sd *sd)
{
        int i;
        for (i = 0; i < 0x80; i++) {
                unsigned char val = 0;
                m5602_read_bridge(sd, i, &val);
                pr_info("ALi m5602 address 0x%x contains 0x%x\n", i, val);
        }
        pr_info("Warning: The ALi m5602 webcam probably won't work until it's power cycled\n");
}

static int m5602_probe_sensor(struct sd *sd)
{
        /* Try the po1030 */
        sd->sensor = &po1030;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the mt9m111 sensor */
        sd->sensor = &mt9m111;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the s5k4aa */
        sd->sensor = &s5k4aa;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the ov9650 */
        sd->sensor = &ov9650;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the ov7660 */
        sd->sensor = &ov7660;
        if (!sd->sensor->probe(sd))
                return 0;

        /* Try the s5k83a */
        sd->sensor = &s5k83a;
        if (!sd->sensor->probe(sd))
                return 0;

        /* More sensor probe function goes here */
        pr_info("Failed to find a sensor\n");
        sd->sensor = NULL;
        return -ENODEV;
}

static int m5602_configure(struct gspca_dev *gspca_dev,
                           const struct usb_device_id *id);

static int m5602_init(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        int err;

        gspca_dbg(gspca_dev, D_CONF, "Initializing ALi m5602 webcam\n");
        /* Run the init sequence */
        err = sd->sensor->init(sd);

        return err;
}

static int m5602_init_controls(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (!sd->sensor->init_controls)
                return 0;

        return sd->sensor->init_controls(sd);
}

static int m5602_start_transfer(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        __u8 *buf = sd->gspca_dev.usb_buf;
        int err;

        /* Send start command to the camera */
        const u8 buffer[4] = {0x13, 0xf9, 0x0f, 0x01};

        if (sd->sensor->start)
                sd->sensor->start(sd);

        memcpy(buf, buffer, sizeof(buffer));
        err = usb_control_msg(gspca_dev->dev,
                              usb_sndctrlpipe(gspca_dev->dev, 0),
                              0x04, 0x40, 0x19, 0x0000, buf,
                              sizeof(buffer), M5602_URB_MSG_TIMEOUT);

        gspca_dbg(gspca_dev, D_STREAM, "Transfer started\n");
        return (err < 0) ? err : 0;
}

static void m5602_urb_complete(struct gspca_dev *gspca_dev,
                                u8 *data, int len)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (len < 6) {
                gspca_dbg(gspca_dev, D_PACK, "Packet is less than 6 bytes\n");
                return;
        }

        /* Frame delimiter: ff xx xx xx ff ff */
        if (data[0] == 0xff && data[4] == 0xff && data[5] == 0xff &&
            data[2] != sd->frame_id) {
                gspca_dbg(gspca_dev, D_FRAM, "Frame delimiter detected\n");
                sd->frame_id = data[2];

                /* Remove the extra fluff appended on each header */
                data += 6;
                len -= 6;

                /* Complete the last frame (if any) */
                gspca_frame_add(gspca_dev, LAST_PACKET,
                                NULL, 0);
                sd->frame_count++;

                /* Create a new frame */
                gspca_frame_add(gspca_dev, FIRST_PACKET, data, len);

                gspca_dbg(gspca_dev, D_FRAM, "Starting new frame %d\n",
                          sd->frame_count);

        } else {
                int cur_frame_len;

                cur_frame_len = gspca_dev->image_len;
                /* Remove urb header */
                data += 4;
                len -= 4;

                if (cur_frame_len + len <= gspca_dev->pixfmt.sizeimage) {
                        gspca_dbg(gspca_dev, D_FRAM, "Continuing frame %d copying %d bytes\n",
                                  sd->frame_count, len);

                        gspca_frame_add(gspca_dev, INTER_PACKET,
                                        data, len);
                } else {
                        /* Add the remaining data up to frame size */
                        gspca_frame_add(gspca_dev, INTER_PACKET, data,
                                gspca_dev->pixfmt.sizeimage - cur_frame_len);
                }
        }
}

static void m5602_stop_transfer(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;

        /* Run the sensor specific end transfer sequence */
        if (sd->sensor->stop)
                sd->sensor->stop(sd);
}

/* sub-driver description */
static const struct sd_desc sd_desc = {
        .name                = MODULE_NAME,
        .config                = m5602_configure,
        .init                = m5602_init,
        .init_controls        = m5602_init_controls,
        .start                = m5602_start_transfer,
        .stopN                = m5602_stop_transfer,
        .pkt_scan        = m5602_urb_complete
};

/* this function is called at probe time */
static int m5602_configure(struct gspca_dev *gspca_dev,
                           const struct usb_device_id *id)
{
        struct sd *sd = (struct sd *) gspca_dev;
        struct cam *cam;
        int err;

        cam = &gspca_dev->cam;

        if (dump_bridge)
                m5602_dump_bridge(sd);

        /* Probe sensor */
        err = m5602_probe_sensor(sd);
        if (err)
                goto fail;

        return 0;

fail:
        gspca_err(gspca_dev, "ALi m5602 webcam failed\n");
        cam->cam_mode = NULL;
        cam->nmodes = 0;

        return err;
}

static int m5602_probe(struct usb_interface *intf,
                       const struct usb_device_id *id)
{
        return gspca_dev_probe(intf, id, &sd_desc, sizeof(struct sd),
                               THIS_MODULE);
}

static void m5602_disconnect(struct usb_interface *intf)
{
        struct gspca_dev *gspca_dev = usb_get_intfdata(intf);
        struct sd *sd = (struct sd *) gspca_dev;

        if (sd->sensor->disconnect)
                sd->sensor->disconnect(sd);

        gspca_disconnect(intf);
}

static struct usb_driver sd_driver = {
        .name = MODULE_NAME,
        .id_table = m5602_table,
        .probe = m5602_probe,
#ifdef CONFIG_PM
        .suspend = gspca_suspend,
        .resume = gspca_resume,
        .reset_resume = gspca_resume,
#endif
        .disconnect = m5602_disconnect
};

module_usb_driver(sd_driver);

MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
MODULE_LICENSE("GPL");
module_param(force_sensor, int, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(force_sensor,
                "forces detection of a sensor, 1 = OV9650, 2 = S5K83A, 3 = S5K4AA, 4 = MT9M111, 5 = PO1030, 6 = OV7660");

module_param(dump_bridge, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dump_bridge, "Dumps all usb bridge registers at startup");

module_param(dump_sensor, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dump_sensor, "Dumps all usb sensor registers at startup providing a sensor is found");
















   15 













   43 
   43 








   42 




   43 

   43 




   42 
   15 


   41 
    9 
    4 
    4 

   41 




   42 

   42 
   42 
   40 


   42 
   41 







    1 
   42 






    4 





    4 

    1 

    1 
    4 
    1 


    4 
    4 




    1 

    4 
    1 
    1 
    1 

    4 
    4 
    4 





    4 
    4 
    4 
    4 

    4 
















































   39 




   38 
   38 
   38 

   39 






   34 



   34 
   34 

   34 
   34 

   34 












   31 













































    9 



    5 



    5 
    5 
    5 
    5 
    5 

    3 
    5 



    1 
    1 

    1 
    1 

    5 
    5 








    4 

    1 
    1 
    1 
    1 








   23 







   23 





   23 






   22 
    3 


   22 
   22 


   22 

   22 

   22 















    3 
   23 






   17 





   17 


    1 

    1 

    1 


   17 

   17 




   17 
   17 



   15 



    7 

    7 

   17 
   17 







   29 




   30 
    6 


    8 


    8 




    8 

    8 

    2 
    8 
    7 

    2 



    7 

    5 





    5 


    3 
    3 






    2 

    2 


    7 

    6 

    7 
    2 





    5 
    3 
    3 
    3 

    5 


   30 








    3 
    3 

    3 
    3 








   10 


   10 
   10 






















   10 















   10 




   10 
   10 











































   10 





   10 
   10 
   10 

































































































   58 

   20 




   56 




   56 





   56 























   56 


   58 








   14 


   14 
   14 
   14 







































   29 

    2 
   29 































   29 
   28 
   29 

    1 
   28 
    5 
   29 












   10 














   11 
   11 





   11 

   11 

   11 














   11 




















































































   26 


   10 

   10 


   10 








   10 







   10 




















   17 




   26 







    5 
    5 

    5 
    5 
    5 

    5 

    5 





















































































    1 

    1 

    1 




    1 


    1 






    1 

    1 


    1 





















   26 

   25 
   26 






   26 

    9 

   26 
   26 
   26 
   10 

   26 



   26 



   26 

   26 







   26 
   26 

   26 



   26 
   26 

   26 
   26 





   21 


   21 






   11 
   11 

   11 


   10 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */

#include <linux/skmsg.h>
#include <linux/skbuff.h>
#include <linux/scatterlist.h>

#include <net/sock.h>
#include <net/tcp.h>
#include <net/tls.h>
#include <trace/events/sock.h>

static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce)
{
        if (msg->sg.end > msg->sg.start &&
            elem_first_coalesce < msg->sg.end)
                return true;

        if (msg->sg.end < msg->sg.start &&
            (elem_first_coalesce > msg->sg.start ||
             elem_first_coalesce < msg->sg.end))
                return true;

        return false;
}

int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
                 int elem_first_coalesce)
{
        struct page_frag *pfrag = sk_page_frag(sk);
        u32 osize = msg->sg.size;
        int ret = 0;

        len -= msg->sg.size;
        while (len > 0) {
                struct scatterlist *sge;
                u32 orig_offset;
                int use, i;

                if (!sk_page_frag_refill(sk, pfrag)) {
                        ret = -ENOMEM;
                        goto msg_trim;
                }

                orig_offset = pfrag->offset;
                use = min_t(int, len, pfrag->size - orig_offset);
                if (!sk_wmem_schedule(sk, use)) {
                        ret = -ENOMEM;
                        goto msg_trim;
                }

                i = msg->sg.end;
                sk_msg_iter_var_prev(i);
                sge = &msg->sg.data[i];

                if (sk_msg_try_coalesce_ok(msg, elem_first_coalesce) &&
                    sg_page(sge) == pfrag->page &&
                    sge->offset + sge->length == orig_offset) {
                        sge->length += use;
                } else {
                        if (sk_msg_full(msg)) {
                                ret = -ENOSPC;
                                break;
                        }

                        sge = &msg->sg.data[msg->sg.end];
                        sg_unmark_end(sge);
                        sg_set_page(sge, pfrag->page, use, orig_offset);
                        get_page(pfrag->page);
                        sk_msg_iter_next(msg, end);
                }

                sk_mem_charge(sk, use);
                msg->sg.size += use;
                pfrag->offset += use;
                len -= use;
        }

        return ret;

msg_trim:
        sk_msg_trim(sk, msg, osize);
        return ret;
}
EXPORT_SYMBOL_GPL(sk_msg_alloc);

int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
                 u32 off, u32 len)
{
        int i = src->sg.start;
        struct scatterlist *sge = sk_msg_elem(src, i);
        struct scatterlist *sgd = NULL;
        u32 sge_len, sge_off;

        while (off) {
                if (sge->length > off)
                        break;
                off -= sge->length;
                sk_msg_iter_var_next(i);
                if (i == src->sg.end && off)
                        return -ENOSPC;
                sge = sk_msg_elem(src, i);
        }

        while (len) {
                sge_len = sge->length - off;
                if (sge_len > len)
                        sge_len = len;

                if (dst->sg.end)
                        sgd = sk_msg_elem(dst, dst->sg.end - 1);

                if (sgd &&
                    (sg_page(sge) == sg_page(sgd)) &&
                    (sg_virt(sge) + off == sg_virt(sgd) + sgd->length)) {
                        sgd->length += sge_len;
                        dst->sg.size += sge_len;
                } else if (!sk_msg_full(dst)) {
                        sge_off = sge->offset + off;
                        sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off);
                } else {
                        return -ENOSPC;
                }

                off = 0;
                len -= sge_len;
                sk_mem_charge(sk, sge_len);
                sk_msg_iter_var_next(i);
                if (i == src->sg.end && len)
                        return -ENOSPC;
                sge = sk_msg_elem(src, i);
        }

        return 0;
}
EXPORT_SYMBOL_GPL(sk_msg_clone);

void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes)
{
        int i = msg->sg.start;

        do {
                struct scatterlist *sge = sk_msg_elem(msg, i);

                if (bytes < sge->length) {
                        sge->length -= bytes;
                        sge->offset += bytes;
                        sk_mem_uncharge(sk, bytes);
                        break;
                }

                sk_mem_uncharge(sk, sge->length);
                bytes -= sge->length;
                sge->length = 0;
                sge->offset = 0;
                sk_msg_iter_var_next(i);
        } while (bytes && i != msg->sg.end);
        msg->sg.start = i;
}
EXPORT_SYMBOL_GPL(sk_msg_return_zero);

void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes)
{
        int i = msg->sg.start;

        do {
                struct scatterlist *sge = &msg->sg.data[i];
                int uncharge = (bytes < sge->length) ? bytes : sge->length;

                sk_mem_uncharge(sk, uncharge);
                bytes -= uncharge;
                sk_msg_iter_var_next(i);
        } while (i != msg->sg.end);
}
EXPORT_SYMBOL_GPL(sk_msg_return);

static int sk_msg_free_elem(struct sock *sk, struct sk_msg *msg, u32 i,
                            bool charge)
{
        struct scatterlist *sge = sk_msg_elem(msg, i);
        u32 len = sge->length;

        /* When the skb owns the memory we free it from consume_skb path. */
        if (!msg->skb) {
                if (charge)
                        sk_mem_uncharge(sk, len);
                put_page(sg_page(sge));
        }
        memset(sge, 0, sizeof(*sge));
        return len;
}

static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i,
                         bool charge)
{
        struct scatterlist *sge = sk_msg_elem(msg, i);
        int freed = 0;

        while (msg->sg.size) {
                msg->sg.size -= sge->length;
                freed += sk_msg_free_elem(sk, msg, i, charge);
                sk_msg_iter_var_next(i);
                sk_msg_check_to_free(msg, i, msg->sg.size);
                sge = sk_msg_elem(msg, i);
        }
        consume_skb(msg->skb);
        sk_msg_init(msg);
        return freed;
}

int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg)
{
        return __sk_msg_free(sk, msg, msg->sg.start, false);
}
EXPORT_SYMBOL_GPL(sk_msg_free_nocharge);

int sk_msg_free(struct sock *sk, struct sk_msg *msg)
{
        return __sk_msg_free(sk, msg, msg->sg.start, true);
}
EXPORT_SYMBOL_GPL(sk_msg_free);

static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg,
                                  u32 bytes, bool charge)
{
        struct scatterlist *sge;
        u32 i = msg->sg.start;

        while (bytes) {
                sge = sk_msg_elem(msg, i);
                if (!sge->length)
                        break;
                if (bytes < sge->length) {
                        if (charge)
                                sk_mem_uncharge(sk, bytes);
                        sge->length -= bytes;
                        sge->offset += bytes;
                        msg->sg.size -= bytes;
                        break;
                }

                msg->sg.size -= sge->length;
                bytes -= sge->length;
                sk_msg_free_elem(sk, msg, i, charge);
                sk_msg_iter_var_next(i);
                sk_msg_check_to_free(msg, i, bytes);
        }
        msg->sg.start = i;
}

void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes)
{
        __sk_msg_free_partial(sk, msg, bytes, true);
}
EXPORT_SYMBOL_GPL(sk_msg_free_partial);

void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
                                  u32 bytes)
{
        __sk_msg_free_partial(sk, msg, bytes, false);
}

void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len)
{
        int trim = msg->sg.size - len;
        u32 i = msg->sg.end;

        if (trim <= 0) {
                WARN_ON(trim < 0);
                return;
        }

        sk_msg_iter_var_prev(i);
        msg->sg.size = len;
        while (msg->sg.data[i].length &&
               trim >= msg->sg.data[i].length) {
                trim -= msg->sg.data[i].length;
                sk_msg_free_elem(sk, msg, i, true);
                sk_msg_iter_var_prev(i);
                if (!trim)
                        goto out;
        }

        msg->sg.data[i].length -= trim;
        sk_mem_uncharge(sk, trim);
        /* Adjust copybreak if it falls into the trimmed part of last buf */
        if (msg->sg.curr == i && msg->sg.copybreak > msg->sg.data[i].length)
                msg->sg.copybreak = msg->sg.data[i].length;
out:
        sk_msg_iter_var_next(i);
        msg->sg.end = i;

        /* If we trim data a full sg elem before curr pointer update
         * copybreak and current so that any future copy operations
         * start at new copy location.
         * However trimmed data that has not yet been used in a copy op
         * does not require an update.
         */
        if (!msg->sg.size) {
                msg->sg.curr = msg->sg.start;
                msg->sg.copybreak = 0;
        } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >=
                   sk_msg_iter_dist(msg->sg.start, msg->sg.end)) {
                sk_msg_iter_var_prev(i);
                msg->sg.curr = i;
                msg->sg.copybreak = msg->sg.data[i].length;
        }
}
EXPORT_SYMBOL_GPL(sk_msg_trim);

int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
                              struct sk_msg *msg, u32 bytes)
{
        int i, maxpages, ret = 0, num_elems = sk_msg_elem_used(msg);
        const int to_max_pages = MAX_MSG_FRAGS;
        struct page *pages[MAX_MSG_FRAGS];
        ssize_t orig, copied, use, offset;

        orig = msg->sg.size;
        while (bytes > 0) {
                i = 0;
                maxpages = to_max_pages - num_elems;
                if (maxpages == 0) {
                        ret = -EFAULT;
                        goto out;
                }

                copied = iov_iter_get_pages2(from, pages, bytes, maxpages,
                                            &offset);
                if (copied <= 0) {
                        ret = -EFAULT;
                        goto out;
                }

                bytes -= copied;
                msg->sg.size += copied;

                while (copied) {
                        use = min_t(int, copied, PAGE_SIZE - offset);
                        sg_set_page(&msg->sg.data[msg->sg.end],
                                    pages[i], use, offset);
                        sg_unmark_end(&msg->sg.data[msg->sg.end]);
                        sk_mem_charge(sk, use);

                        offset = 0;
                        copied -= use;
                        sk_msg_iter_next(msg, end);
                        num_elems++;
                        i++;
                }
                /* When zerocopy is mixed with sk_msg_*copy* operations we
                 * may have a copybreak set in this case clear and prefer
                 * zerocopy remainder when possible.
                 */
                msg->sg.copybreak = 0;
                msg->sg.curr = msg->sg.end;
        }
out:
        /* Revert iov_iter updates, msg will need to use 'trim' later if it
         * also needs to be cleared.
         */
        if (ret)
                iov_iter_revert(from, msg->sg.size - orig);
        return ret;
}
EXPORT_SYMBOL_GPL(sk_msg_zerocopy_from_iter);

int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
                             struct sk_msg *msg, u32 bytes)
{
        int ret = -ENOSPC, i = msg->sg.curr;
        u32 copy, buf_size, copied = 0;
        struct scatterlist *sge;
        void *to;

        do {
                sge = sk_msg_elem(msg, i);
                /* This is possible if a trim operation shrunk the buffer */
                if (msg->sg.copybreak >= sge->length) {
                        msg->sg.copybreak = 0;
                        sk_msg_iter_var_next(i);
                        if (i == msg->sg.end)
                                break;
                        sge = sk_msg_elem(msg, i);
                }

                buf_size = sge->length - msg->sg.copybreak;
                copy = (buf_size > bytes) ? bytes : buf_size;
                to = sg_virt(sge) + msg->sg.copybreak;
                msg->sg.copybreak += copy;
                if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY)
                        ret = copy_from_iter_nocache(to, copy, from);
                else
                        ret = copy_from_iter(to, copy, from);
                if (ret != copy) {
                        ret = -EFAULT;
                        goto out;
                }
                bytes -= copy;
                copied += copy;
                if (!bytes)
                        break;
                msg->sg.copybreak = 0;
                sk_msg_iter_var_next(i);
        } while (i != msg->sg.end);
out:
        msg->sg.curr = i;
        return (ret < 0) ? ret : copied;
}
EXPORT_SYMBOL_GPL(sk_msg_memcopy_from_iter);

/* Receive sk_msg from psock->ingress_msg to @msg. */
int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
                   int len, int flags)
{
        struct iov_iter *iter = &msg->msg_iter;
        int peek = flags & MSG_PEEK;
        struct sk_msg *msg_rx;
        int i, copied = 0;

        msg_rx = sk_psock_peek_msg(psock);
        while (copied != len) {
                struct scatterlist *sge;

                if (unlikely(!msg_rx))
                        break;

                i = msg_rx->sg.start;
                do {
                        struct page *page;
                        int copy;

                        sge = sk_msg_elem(msg_rx, i);
                        copy = sge->length;
                        page = sg_page(sge);
                        if (copied + copy > len)
                                copy = len - copied;
                        if (copy)
                                copy = copy_page_to_iter(page, sge->offset, copy, iter);
                        if (!copy) {
                                copied = copied ? copied : -EFAULT;
                                goto out;
                        }

                        copied += copy;
                        if (likely(!peek)) {
                                sge->offset += copy;
                                sge->length -= copy;
                                if (!msg_rx->skb) {
                                        sk_mem_uncharge(sk, copy);
                                        atomic_sub(copy, &sk->sk_rmem_alloc);
                                }
                                msg_rx->sg.size -= copy;

                                if (!sge->length) {
                                        sk_msg_iter_var_next(i);
                                        if (!msg_rx->skb)
                                                put_page(page);
                                }
                        } else {
                                /* Lets not optimize peek case if copy_page_to_iter
                                 * didn't copy the entire length lets just break.
                                 */
                                if (copy != sge->length)
                                        goto out;
                                sk_msg_iter_var_next(i);
                        }

                        if (copied == len)
                                break;
                } while ((i != msg_rx->sg.end) && !sg_is_last(sge));

                if (unlikely(peek)) {
                        msg_rx = sk_psock_next_msg(psock, msg_rx);
                        if (!msg_rx)
                                break;
                        continue;
                }

                msg_rx->sg.start = i;
                if (!sge->length && (i == msg_rx->sg.end || sg_is_last(sge))) {
                        msg_rx = sk_psock_dequeue_msg(psock);
                        kfree_sk_msg(msg_rx);
                }
                msg_rx = sk_psock_peek_msg(psock);
        }
out:
        return copied;
}
EXPORT_SYMBOL_GPL(sk_msg_recvmsg);

bool sk_msg_is_readable(struct sock *sk)
{
        struct sk_psock *psock;
        bool empty = true;

        rcu_read_lock();
        psock = sk_psock(sk);
        if (likely(psock))
                empty = list_empty(&psock->ingress_msg);
        rcu_read_unlock();
        return !empty;
}
EXPORT_SYMBOL_GPL(sk_msg_is_readable);

static struct sk_msg *alloc_sk_msg(gfp_t gfp)
{
        struct sk_msg *msg;

        msg = kzalloc(sizeof(*msg), gfp | __GFP_NOWARN);
        if (unlikely(!msg))
                return NULL;
        sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);
        return msg;
}

static struct sk_msg *sk_psock_create_ingress_msg(struct sock *sk,
                                                  struct sk_buff *skb)
{
        if (atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf)
                return NULL;

        if (!sk_rmem_schedule(sk, skb, skb->truesize))
                return NULL;

        return alloc_sk_msg(GFP_KERNEL);
}

static int sk_psock_skb_ingress_enqueue(struct sk_buff *skb,
                                        u32 off, u32 len,
                                        struct sk_psock *psock,
                                        struct sock *sk,
                                        struct sk_msg *msg)
{
        int num_sge, copied;

        num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
        if (num_sge < 0) {
                /* skb linearize may fail with ENOMEM, but lets simply try again
                 * later if this happens. Under memory pressure we don't want to
                 * drop the skb. We need to linearize the skb so that the mapping
                 * in skb_to_sgvec can not error.
                 */
                if (skb_linearize(skb))
                        return -EAGAIN;

                num_sge = skb_to_sgvec(skb, msg->sg.data, off, len);
                if (unlikely(num_sge < 0))
                        return num_sge;
        }

        copied = len;
        msg->sg.start = 0;
        msg->sg.size = copied;
        msg->sg.end = num_sge;
        msg->skb = skb;

        sk_psock_queue_msg(psock, msg);
        sk_psock_data_ready(sk, psock);
        return copied;
}

static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
                                     u32 off, u32 len);

static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb,
                                u32 off, u32 len)
{
        struct sock *sk = psock->sk;
        struct sk_msg *msg;
        int err;

        /* If we are receiving on the same sock skb->sk is already assigned,
         * skip memory accounting and owner transition seeing it already set
         * correctly.
         */
        if (unlikely(skb->sk == sk))
                return sk_psock_skb_ingress_self(psock, skb, off, len);
        msg = sk_psock_create_ingress_msg(sk, skb);
        if (!msg)
                return -EAGAIN;

        /* This will transition ownership of the data from the socket where
         * the BPF program was run initiating the redirect to the socket
         * we will eventually receive this data on. The data will be released
         * from skb_consume found in __tcp_bpf_recvmsg() after its been copied
         * into user buffers.
         */
        skb_set_owner_r(skb, sk);
        err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
        if (err < 0)
                kfree(msg);
        return err;
}

/* Puts an skb on the ingress queue of the socket already assigned to the
 * skb. In this case we do not need to check memory limits or skb_set_owner_r
 * because the skb is already accounted for here.
 */
static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
                                     u32 off, u32 len)
{
        struct sk_msg *msg = alloc_sk_msg(GFP_ATOMIC);
        struct sock *sk = psock->sk;
        int err;

        if (unlikely(!msg))
                return -EAGAIN;
        skb_set_owner_r(skb, sk);
        err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg);
        if (err < 0)
                kfree(msg);
        return err;
}

static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,
                               u32 off, u32 len, bool ingress)
{
        int err = 0;

        if (!ingress) {
                if (!sock_writeable(psock->sk))
                        return -EAGAIN;
                return skb_send_sock(psock->sk, skb, off, len);
        }
        skb_get(skb);
        err = sk_psock_skb_ingress(psock, skb, off, len);
        if (err < 0)
                kfree_skb(skb);
        return err;
}

static void sk_psock_skb_state(struct sk_psock *psock,
                               struct sk_psock_work_state *state,
                               int len, int off)
{
        spin_lock_bh(&psock->ingress_lock);
        if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
                state->len = len;
                state->off = off;
        }
        spin_unlock_bh(&psock->ingress_lock);
}

static void sk_psock_backlog(struct work_struct *work)
{
        struct delayed_work *dwork = to_delayed_work(work);
        struct sk_psock *psock = container_of(dwork, struct sk_psock, work);
        struct sk_psock_work_state *state = &psock->work_state;
        struct sk_buff *skb = NULL;
        u32 len = 0, off = 0;
        bool ingress;
        int ret;

        mutex_lock(&psock->work_mutex);
        if (unlikely(state->len)) {
                len = state->len;
                off = state->off;
        }

        while ((skb = skb_peek(&psock->ingress_skb))) {
                len = skb->len;
                off = 0;
                if (skb_bpf_strparser(skb)) {
                        struct strp_msg *stm = strp_msg(skb);

                        off = stm->offset;
                        len = stm->full_len;
                }
                ingress = skb_bpf_ingress(skb);
                skb_bpf_redirect_clear(skb);
                do {
                        ret = -EIO;
                        if (!sock_flag(psock->sk, SOCK_DEAD))
                                ret = sk_psock_handle_skb(psock, skb, off,
                                                          len, ingress);
                        if (ret <= 0) {
                                if (ret == -EAGAIN) {
                                        sk_psock_skb_state(psock, state, len, off);

                                        /* Delay slightly to prioritize any
                                         * other work that might be here.
                                         */
                                        if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
                                                schedule_delayed_work(&psock->work, 1);
                                        goto end;
                                }
                                /* Hard errors break pipe and stop xmit. */
                                sk_psock_report_error(psock, ret ? -ret : EPIPE);
                                sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
                                goto end;
                        }
                        off += ret;
                        len -= ret;
                } while (len);

                skb = skb_dequeue(&psock->ingress_skb);
                kfree_skb(skb);
        }
end:
        mutex_unlock(&psock->work_mutex);
}

struct sk_psock *sk_psock_init(struct sock *sk, int node)
{
        struct sk_psock *psock;
        struct proto *prot;

        write_lock_bh(&sk->sk_callback_lock);

        if (sk_is_inet(sk) && inet_csk_has_ulp(sk)) {
                psock = ERR_PTR(-EINVAL);
                goto out;
        }

        if (sk->sk_user_data) {
                psock = ERR_PTR(-EBUSY);
                goto out;
        }

        psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node);
        if (!psock) {
                psock = ERR_PTR(-ENOMEM);
                goto out;
        }

        prot = READ_ONCE(sk->sk_prot);
        psock->sk = sk;
        psock->eval = __SK_NONE;
        psock->sk_proto = prot;
        psock->saved_unhash = prot->unhash;
        psock->saved_destroy = prot->destroy;
        psock->saved_close = prot->close;
        psock->saved_write_space = sk->sk_write_space;

        INIT_LIST_HEAD(&psock->link);
        spin_lock_init(&psock->link_lock);

        INIT_DELAYED_WORK(&psock->work, sk_psock_backlog);
        mutex_init(&psock->work_mutex);
        INIT_LIST_HEAD(&psock->ingress_msg);
        spin_lock_init(&psock->ingress_lock);
        skb_queue_head_init(&psock->ingress_skb);

        sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED);
        refcount_set(&psock->refcnt, 1);

        __rcu_assign_sk_user_data_with_flags(sk, psock,
                                             SK_USER_DATA_NOCOPY |
                                             SK_USER_DATA_PSOCK);
        sock_hold(sk);

out:
        write_unlock_bh(&sk->sk_callback_lock);
        return psock;
}
EXPORT_SYMBOL_GPL(sk_psock_init);

struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock)
{
        struct sk_psock_link *link;

        spin_lock_bh(&psock->link_lock);
        link = list_first_entry_or_null(&psock->link, struct sk_psock_link,
                                        list);
        if (link)
                list_del(&link->list);
        spin_unlock_bh(&psock->link_lock);
        return link;
}

static void __sk_psock_purge_ingress_msg(struct sk_psock *psock)
{
        struct sk_msg *msg, *tmp;

        list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) {
                list_del(&msg->list);
                if (!msg->skb)
                        atomic_sub(msg->sg.size, &psock->sk->sk_rmem_alloc);
                sk_msg_free(psock->sk, msg);
                kfree(msg);
        }
}

static void __sk_psock_zap_ingress(struct sk_psock *psock)
{
        struct sk_buff *skb;

        while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) {
                skb_bpf_redirect_clear(skb);
                sock_drop(psock->sk, skb);
        }
        __sk_psock_purge_ingress_msg(psock);
}

static void sk_psock_link_destroy(struct sk_psock *psock)
{
        struct sk_psock_link *link, *tmp;

        list_for_each_entry_safe(link, tmp, &psock->link, list) {
                list_del(&link->list);
                sk_psock_free_link(link);
        }
}

void sk_psock_stop(struct sk_psock *psock)
{
        spin_lock_bh(&psock->ingress_lock);
        sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
        sk_psock_cork_free(psock);
        spin_unlock_bh(&psock->ingress_lock);
}

static void sk_psock_done_strp(struct sk_psock *psock);

static void sk_psock_destroy(struct work_struct *work)
{
        struct sk_psock *psock = container_of(to_rcu_work(work),
                                              struct sk_psock, rwork);
        /* No sk_callback_lock since already detached. */

        sk_psock_done_strp(psock);

        cancel_delayed_work_sync(&psock->work);
        __sk_psock_zap_ingress(psock);
        mutex_destroy(&psock->work_mutex);

        psock_progs_drop(&psock->progs);

        sk_psock_link_destroy(psock);
        sk_psock_cork_free(psock);

        if (psock->sk_redir)
                sock_put(psock->sk_redir);
        if (psock->sk_pair)
                sock_put(psock->sk_pair);
        sock_put(psock->sk);
        kfree(psock);
}

void sk_psock_drop(struct sock *sk, struct sk_psock *psock)
{
        write_lock_bh(&sk->sk_callback_lock);
        sk_psock_restore_proto(sk, psock);
        rcu_assign_sk_user_data(sk, NULL);
        if (psock->progs.stream_parser)
                sk_psock_stop_strp(sk, psock);
        else if (psock->progs.stream_verdict || psock->progs.skb_verdict)
                sk_psock_stop_verdict(sk, psock);
        write_unlock_bh(&sk->sk_callback_lock);

        sk_psock_stop(psock);

        INIT_RCU_WORK(&psock->rwork, sk_psock_destroy);
        queue_rcu_work(system_wq, &psock->rwork);
}
EXPORT_SYMBOL_GPL(sk_psock_drop);

static int sk_psock_map_verd(int verdict, bool redir)
{
        switch (verdict) {
        case SK_PASS:
                return redir ? __SK_REDIRECT : __SK_PASS;
        case SK_DROP:
        default:
                break;
        }

        return __SK_DROP;
}

int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
                         struct sk_msg *msg)
{
        struct bpf_prog *prog;
        int ret;

        rcu_read_lock();
        prog = READ_ONCE(psock->progs.msg_parser);
        if (unlikely(!prog)) {
                ret = __SK_PASS;
                goto out;
        }

        sk_msg_compute_data_pointers(msg);
        msg->sk = sk;
        ret = bpf_prog_run_pin_on_cpu(prog, msg);
        ret = sk_psock_map_verd(ret, msg->sk_redir);
        psock->apply_bytes = msg->apply_bytes;
        if (ret == __SK_REDIRECT) {
                if (psock->sk_redir) {
                        sock_put(psock->sk_redir);
                        psock->sk_redir = NULL;
                }
                if (!msg->sk_redir) {
                        ret = __SK_DROP;
                        goto out;
                }
                psock->redir_ingress = sk_msg_to_ingress(msg);
                psock->sk_redir = msg->sk_redir;
                sock_hold(psock->sk_redir);
        }
out:
        rcu_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(sk_psock_msg_verdict);

static int sk_psock_skb_redirect(struct sk_psock *from, struct sk_buff *skb)
{
        struct sk_psock *psock_other;
        struct sock *sk_other;

        sk_other = skb_bpf_redirect_fetch(skb);
        /* This error is a buggy BPF program, it returned a redirect
         * return code, but then didn't set a redirect interface.
         */
        if (unlikely(!sk_other)) {
                skb_bpf_redirect_clear(skb);
                sock_drop(from->sk, skb);
                return -EIO;
        }
        psock_other = sk_psock(sk_other);
        /* This error indicates the socket is being torn down or had another
         * error that caused the pipe to break. We can't send a packet on
         * a socket that is in this state so we drop the skb.
         */
        if (!psock_other || sock_flag(sk_other, SOCK_DEAD)) {
                skb_bpf_redirect_clear(skb);
                sock_drop(from->sk, skb);
                return -EIO;
        }
        spin_lock_bh(&psock_other->ingress_lock);
        if (!sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
                spin_unlock_bh(&psock_other->ingress_lock);
                skb_bpf_redirect_clear(skb);
                sock_drop(from->sk, skb);
                return -EIO;
        }

        skb_queue_tail(&psock_other->ingress_skb, skb);
        schedule_delayed_work(&psock_other->work, 0);
        spin_unlock_bh(&psock_other->ingress_lock);
        return 0;
}

static void sk_psock_tls_verdict_apply(struct sk_buff *skb,
                                       struct sk_psock *from, int verdict)
{
        switch (verdict) {
        case __SK_REDIRECT:
                sk_psock_skb_redirect(from, skb);
                break;
        case __SK_PASS:
        case __SK_DROP:
        default:
                break;
        }
}

int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb)
{
        struct bpf_prog *prog;
        int ret = __SK_PASS;

        rcu_read_lock();
        prog = READ_ONCE(psock->progs.stream_verdict);
        if (likely(prog)) {
                skb->sk = psock->sk;
                skb_dst_drop(skb);
                skb_bpf_redirect_clear(skb);
                ret = bpf_prog_run_pin_on_cpu(prog, skb);
                ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
                skb->sk = NULL;
        }
        sk_psock_tls_verdict_apply(skb, psock, ret);
        rcu_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(sk_psock_tls_strp_read);

static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb,
                                  int verdict)
{
        struct sock *sk_other;
        int err = 0;
        u32 len, off;

        switch (verdict) {
        case __SK_PASS:
                err = -EIO;
                sk_other = psock->sk;
                if (sock_flag(sk_other, SOCK_DEAD) ||
                    !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
                        goto out_free;

                skb_bpf_set_ingress(skb);

                /* If the queue is empty then we can submit directly
                 * into the msg queue. If its not empty we have to
                 * queue work otherwise we may get OOO data. Otherwise,
                 * if sk_psock_skb_ingress errors will be handled by
                 * retrying later from workqueue.
                 */
                if (skb_queue_empty(&psock->ingress_skb)) {
                        len = skb->len;
                        off = 0;
                        if (skb_bpf_strparser(skb)) {
                                struct strp_msg *stm = strp_msg(skb);

                                off = stm->offset;
                                len = stm->full_len;
                        }
                        err = sk_psock_skb_ingress_self(psock, skb, off, len);
                }
                if (err < 0) {
                        spin_lock_bh(&psock->ingress_lock);
                        if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
                                skb_queue_tail(&psock->ingress_skb, skb);
                                schedule_delayed_work(&psock->work, 0);
                                err = 0;
                        }
                        spin_unlock_bh(&psock->ingress_lock);
                        if (err < 0)
                                goto out_free;
                }
                break;
        case __SK_REDIRECT:
                tcp_eat_skb(psock->sk, skb);
                err = sk_psock_skb_redirect(psock, skb);
                break;
        case __SK_DROP:
        default:
out_free:
                skb_bpf_redirect_clear(skb);
                tcp_eat_skb(psock->sk, skb);
                sock_drop(psock->sk, skb);
        }

        return err;
}

static void sk_psock_write_space(struct sock *sk)
{
        struct sk_psock *psock;
        void (*write_space)(struct sock *sk) = NULL;

        rcu_read_lock();
        psock = sk_psock(sk);
        if (likely(psock)) {
                if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED))
                        schedule_delayed_work(&psock->work, 0);
                write_space = psock->saved_write_space;
        }
        rcu_read_unlock();
        if (write_space)
                write_space(sk);
}

#if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
static void sk_psock_strp_read(struct strparser *strp, struct sk_buff *skb)
{
        struct sk_psock *psock;
        struct bpf_prog *prog;
        int ret = __SK_DROP;
        struct sock *sk;

        rcu_read_lock();
        sk = strp->sk;
        psock = sk_psock(sk);
        if (unlikely(!psock)) {
                sock_drop(sk, skb);
                goto out;
        }
        prog = READ_ONCE(psock->progs.stream_verdict);
        if (likely(prog)) {
                skb->sk = sk;
                skb_dst_drop(skb);
                skb_bpf_redirect_clear(skb);
                ret = bpf_prog_run_pin_on_cpu(prog, skb);
                skb_bpf_set_strparser(skb);
                ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
                skb->sk = NULL;
        }
        sk_psock_verdict_apply(psock, skb, ret);
out:
        rcu_read_unlock();
}

static int sk_psock_strp_read_done(struct strparser *strp, int err)
{
        return err;
}

static int sk_psock_strp_parse(struct strparser *strp, struct sk_buff *skb)
{
        struct sk_psock *psock = container_of(strp, struct sk_psock, strp);
        struct bpf_prog *prog;
        int ret = skb->len;

        rcu_read_lock();
        prog = READ_ONCE(psock->progs.stream_parser);
        if (likely(prog)) {
                skb->sk = psock->sk;
                ret = bpf_prog_run_pin_on_cpu(prog, skb);
                skb->sk = NULL;
        }
        rcu_read_unlock();
        return ret;
}

/* Called with socket lock held. */
static void sk_psock_strp_data_ready(struct sock *sk)
{
        struct sk_psock *psock;

        trace_sk_data_ready(sk);

        rcu_read_lock();
        psock = sk_psock(sk);
        if (likely(psock)) {
                if (tls_sw_has_ctx_rx(sk)) {
                        psock->saved_data_ready(sk);
                } else {
                        read_lock_bh(&sk->sk_callback_lock);
                        strp_data_ready(&psock->strp);
                        read_unlock_bh(&sk->sk_callback_lock);
                }
        }
        rcu_read_unlock();
}

int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
{
        int ret;

        static const struct strp_callbacks cb = {
                .rcv_msg        = sk_psock_strp_read,
                .read_sock_done        = sk_psock_strp_read_done,
                .parse_msg        = sk_psock_strp_parse,
        };

        ret = strp_init(&psock->strp, sk, &cb);
        if (!ret)
                sk_psock_set_state(psock, SK_PSOCK_RX_STRP_ENABLED);

        return ret;
}

void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
{
        if (psock->saved_data_ready)
                return;

        psock->saved_data_ready = sk->sk_data_ready;
        sk->sk_data_ready = sk_psock_strp_data_ready;
        sk->sk_write_space = sk_psock_write_space;
}

void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
{
        psock_set_prog(&psock->progs.stream_parser, NULL);

        if (!psock->saved_data_ready)
                return;

        sk->sk_data_ready = psock->saved_data_ready;
        psock->saved_data_ready = NULL;
        strp_stop(&psock->strp);
}

static void sk_psock_done_strp(struct sk_psock *psock)
{
        /* Parser has been stopped */
        if (sk_psock_test_state(psock, SK_PSOCK_RX_STRP_ENABLED))
                strp_done(&psock->strp);
}
#else
static void sk_psock_done_strp(struct sk_psock *psock)
{
}
#endif /* CONFIG_BPF_STREAM_PARSER */

static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb)
{
        struct sk_psock *psock;
        struct bpf_prog *prog;
        int ret = __SK_DROP;
        int len = skb->len;

        rcu_read_lock();
        psock = sk_psock(sk);
        if (unlikely(!psock)) {
                len = 0;
                tcp_eat_skb(sk, skb);
                sock_drop(sk, skb);
                goto out;
        }
        prog = READ_ONCE(psock->progs.stream_verdict);
        if (!prog)
                prog = READ_ONCE(psock->progs.skb_verdict);
        if (likely(prog)) {
                skb_dst_drop(skb);
                skb_bpf_redirect_clear(skb);
                ret = bpf_prog_run_pin_on_cpu(prog, skb);
                ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb));
        }
        ret = sk_psock_verdict_apply(psock, skb, ret);
        if (ret < 0)
                len = ret;
out:
        rcu_read_unlock();
        return len;
}

static void sk_psock_verdict_data_ready(struct sock *sk)
{
        struct socket *sock = sk->sk_socket;
        const struct proto_ops *ops;
        int copied;

        trace_sk_data_ready(sk);

        if (unlikely(!sock))
                return;
        ops = READ_ONCE(sock->ops);
        if (!ops || !ops->read_skb)
                return;
        copied = ops->read_skb(sk, sk_psock_verdict_recv);
        if (copied >= 0) {
                struct sk_psock *psock;

                rcu_read_lock();
                psock = sk_psock(sk);
                if (psock)
                        sk_psock_data_ready(sk, psock);
                rcu_read_unlock();
        }
}

void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
{
        if (psock->saved_data_ready)
                return;

        psock->saved_data_ready = sk->sk_data_ready;
        sk->sk_data_ready = sk_psock_verdict_data_ready;
        sk->sk_write_space = sk_psock_write_space;
}

void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
{
        psock_set_prog(&psock->progs.stream_verdict, NULL);
        psock_set_prog(&psock->progs.skb_verdict, NULL);

        if (!psock->saved_data_ready)
                return;

        sk->sk_data_ready = psock->saved_data_ready;
        psock->saved_data_ready = NULL;
}
























































































































   13 






   12 


    8 

    5 





























   62 
   62 


































   75 

































































   13 
















   71 
   13 


   13 

   13 
   13 




    7 

    7 


    7 




















    8 







    8 


    8 



    4 

    8 


    8 

    1 













    8 





    8 



    8 
    8 


    7 




    9 

    9 






    3 




    2 




    3 



































































































































    2 























































    5 
    1 

    5 







    5 

    5 
    5 
    5 
    1 







    1 

    5 
    5 



















































































































































































   69 



























































   44 




   71 







   71 
   70 


   71 


   71 



   70 
   69 


   71 



















































































   35 





























   89 


   90 


   90 


   90 


   90 



   89 


   90 








   89 















































   62 


   62 

   62 










   62 

   62 



   62 



   62 










   62 
   62 










   62 











   62 





   20 

   20 


   20 








   20 









   43 
   43 









    9 



    9 












    9 


    9 


    9 
    9 

    9 
    9 



    9 












   13 


   13 

   13 


   13 







    4 






    4 
















    4 




    3 



    4 


    4 






   13 






   12 






   12 




   12 


    9 



    5 
    5 
    5 

    5 



    4 
    4 
    4 

    3 







   10 






    3 



    3 




    3 


    3 
    2 
    2 


    3 







    7 






    6 



    4 
    4 
    3 
    4 





    1 













    6 

    7 









































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2006 - 2007 Ivo van Doorn
 * Copyright (C) 2007 Dmitry Torokhov
 * Copyright 2009 Johannes Berg <johannes@sipsolutions.net>
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/workqueue.h>
#include <linux/capability.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/rfkill.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/device.h>
#include <linux/miscdevice.h>
#include <linux/wait.h>
#include <linux/poll.h>
#include <linux/fs.h>
#include <linux/slab.h>

#include "rfkill.h"

#define POLL_INTERVAL                (5 * HZ)

#define RFKILL_BLOCK_HW                BIT(0)
#define RFKILL_BLOCK_SW                BIT(1)
#define RFKILL_BLOCK_SW_PREV        BIT(2)
#define RFKILL_BLOCK_ANY        (RFKILL_BLOCK_HW |\
                                 RFKILL_BLOCK_SW |\
                                 RFKILL_BLOCK_SW_PREV)
#define RFKILL_BLOCK_SW_SETCALL        BIT(31)

struct rfkill {
        spinlock_t                lock;

        enum rfkill_type        type;

        unsigned long                state;
        unsigned long                hard_block_reasons;

        u32                        idx;

        bool                        registered;
        bool                        persistent;
        bool                        polling_paused;
        bool                        suspended;
        bool                        need_sync;

        const struct rfkill_ops        *ops;
        void                        *data;

#ifdef CONFIG_RFKILL_LEDS
        struct led_trigger        led_trigger;
        const char                *ledtrigname;
#endif

        struct device                dev;
        struct list_head        node;

        struct delayed_work        poll_work;
        struct work_struct        uevent_work;
        struct work_struct        sync_work;
        char                        name[];
};
#define to_rfkill(d)        container_of(d, struct rfkill, dev)

struct rfkill_int_event {
        struct list_head        list;
        struct rfkill_event_ext        ev;
};

struct rfkill_data {
        struct list_head        list;
        struct list_head        events;
        struct mutex                mtx;
        wait_queue_head_t        read_wait;
        bool                        input_handler;
        u8                        max_size;
};


MODULE_AUTHOR("Ivo van Doorn <IvDoorn@gmail.com>");
MODULE_AUTHOR("Johannes Berg <johannes@sipsolutions.net>");
MODULE_DESCRIPTION("RF switch support");
MODULE_LICENSE("GPL");


/*
 * The locking here should be made much smarter, we currently have
 * a bit of a stupid situation because drivers might want to register
 * the rfkill struct under their own lock, and take this lock during
 * rfkill method calls -- which will cause an AB-BA deadlock situation.
 *
 * To fix that, we need to rework this code here to be mostly lock-free
 * and only use the mutex for list manipulations, not to protect the
 * various other global variables. Then we can avoid holding the mutex
 * around driver operations, and all is happy.
 */
static LIST_HEAD(rfkill_list);        /* list of registered rf switches */
static DEFINE_MUTEX(rfkill_global_mutex);
static LIST_HEAD(rfkill_fds);        /* list of open fds of /dev/rfkill */

static unsigned int rfkill_default_state = 1;
module_param_named(default_state, rfkill_default_state, uint, 0444);
MODULE_PARM_DESC(default_state,
                 "Default initial state for all radio types, 0 = radio off");

static struct {
        bool cur, sav;
} rfkill_global_states[NUM_RFKILL_TYPES];

static bool rfkill_epo_lock_active;


#ifdef CONFIG_RFKILL_LEDS
static void rfkill_led_trigger_event(struct rfkill *rfkill)
{
        struct led_trigger *trigger;

        if (!rfkill->registered)
                return;

        trigger = &rfkill->led_trigger;

        if (rfkill->state & RFKILL_BLOCK_ANY)
                led_trigger_event(trigger, LED_OFF);
        else
                led_trigger_event(trigger, LED_FULL);
}

static int rfkill_led_trigger_activate(struct led_classdev *led)
{
        struct rfkill *rfkill;

        rfkill = container_of(led->trigger, struct rfkill, led_trigger);

        rfkill_led_trigger_event(rfkill);

        return 0;
}

const char *rfkill_get_led_trigger_name(struct rfkill *rfkill)
{
        return rfkill->led_trigger.name;
}
EXPORT_SYMBOL(rfkill_get_led_trigger_name);

void rfkill_set_led_trigger_name(struct rfkill *rfkill, const char *name)
{
        BUG_ON(!rfkill);

        rfkill->ledtrigname = name;
}
EXPORT_SYMBOL(rfkill_set_led_trigger_name);

static int rfkill_led_trigger_register(struct rfkill *rfkill)
{
        rfkill->led_trigger.name = rfkill->ledtrigname
                                        ? : dev_name(&rfkill->dev);
        rfkill->led_trigger.activate = rfkill_led_trigger_activate;
        return led_trigger_register(&rfkill->led_trigger);
}

static void rfkill_led_trigger_unregister(struct rfkill *rfkill)
{
        led_trigger_unregister(&rfkill->led_trigger);
}

static struct led_trigger rfkill_any_led_trigger;
static struct led_trigger rfkill_none_led_trigger;
static struct work_struct rfkill_global_led_trigger_work;

static void rfkill_global_led_trigger_worker(struct work_struct *work)
{
        enum led_brightness brightness = LED_OFF;
        struct rfkill *rfkill;

        mutex_lock(&rfkill_global_mutex);
        list_for_each_entry(rfkill, &rfkill_list, node) {
                if (!(rfkill->state & RFKILL_BLOCK_ANY)) {
                        brightness = LED_FULL;
                        break;
                }
        }
        mutex_unlock(&rfkill_global_mutex);

        led_trigger_event(&rfkill_any_led_trigger, brightness);
        led_trigger_event(&rfkill_none_led_trigger,
                          brightness == LED_OFF ? LED_FULL : LED_OFF);
}

static void rfkill_global_led_trigger_event(void)
{
        schedule_work(&rfkill_global_led_trigger_work);
}

static int rfkill_global_led_trigger_register(void)
{
        int ret;

        INIT_WORK(&rfkill_global_led_trigger_work,
                        rfkill_global_led_trigger_worker);

        rfkill_any_led_trigger.name = "rfkill-any";
        ret = led_trigger_register(&rfkill_any_led_trigger);
        if (ret)
                return ret;

        rfkill_none_led_trigger.name = "rfkill-none";
        ret = led_trigger_register(&rfkill_none_led_trigger);
        if (ret)
                led_trigger_unregister(&rfkill_any_led_trigger);
        else
                /* Delay activation until all global triggers are registered */
                rfkill_global_led_trigger_event();

        return ret;
}

static void rfkill_global_led_trigger_unregister(void)
{
        led_trigger_unregister(&rfkill_none_led_trigger);
        led_trigger_unregister(&rfkill_any_led_trigger);
        cancel_work_sync(&rfkill_global_led_trigger_work);
}
#else
static void rfkill_led_trigger_event(struct rfkill *rfkill)
{
}

static inline int rfkill_led_trigger_register(struct rfkill *rfkill)
{
        return 0;
}

static inline void rfkill_led_trigger_unregister(struct rfkill *rfkill)
{
}

static void rfkill_global_led_trigger_event(void)
{
}

static int rfkill_global_led_trigger_register(void)
{
        return 0;
}

static void rfkill_global_led_trigger_unregister(void)
{
}
#endif /* CONFIG_RFKILL_LEDS */

static void rfkill_fill_event(struct rfkill_event_ext *ev,
                              struct rfkill *rfkill,
                              enum rfkill_operation op)
{
        unsigned long flags;

        ev->idx = rfkill->idx;
        ev->type = rfkill->type;
        ev->op = op;

        spin_lock_irqsave(&rfkill->lock, flags);
        ev->hard = !!(rfkill->state & RFKILL_BLOCK_HW);
        ev->soft = !!(rfkill->state & (RFKILL_BLOCK_SW |
                                        RFKILL_BLOCK_SW_PREV));
        ev->hard_block_reasons = rfkill->hard_block_reasons;
        spin_unlock_irqrestore(&rfkill->lock, flags);
}

static void rfkill_send_events(struct rfkill *rfkill, enum rfkill_operation op)
{
        struct rfkill_data *data;
        struct rfkill_int_event *ev;

        list_for_each_entry(data, &rfkill_fds, list) {
                ev = kzalloc(sizeof(*ev), GFP_KERNEL);
                if (!ev)
                        continue;
                rfkill_fill_event(&ev->ev, rfkill, op);
                mutex_lock(&data->mtx);
                list_add_tail(&ev->list, &data->events);
                mutex_unlock(&data->mtx);
                wake_up_interruptible(&data->read_wait);
        }
}

static void rfkill_event(struct rfkill *rfkill)
{
        if (!rfkill->registered)
                return;

        kobject_uevent(&rfkill->dev.kobj, KOBJ_CHANGE);

        /* also send event to /dev/rfkill */
        rfkill_send_events(rfkill, RFKILL_OP_CHANGE);
}

/**
 * rfkill_set_block - wrapper for set_block method
 *
 * @rfkill: the rfkill struct to use
 * @blocked: the new software state
 *
 * Calls the set_block method (when applicable) and handles notifications
 * etc. as well.
 */
static void rfkill_set_block(struct rfkill *rfkill, bool blocked)
{
        unsigned long flags;
        bool prev, curr;
        int err;

        if (unlikely(rfkill->dev.power.power_state.event & PM_EVENT_SLEEP))
                return;

        /*
         * Some platforms (...!) generate input events which affect the
         * _hard_ kill state -- whenever something tries to change the
         * current software state query the hardware state too.
         */
        if (rfkill->ops->query)
                rfkill->ops->query(rfkill, rfkill->data);

        spin_lock_irqsave(&rfkill->lock, flags);
        prev = rfkill->state & RFKILL_BLOCK_SW;

        if (prev)
                rfkill->state |= RFKILL_BLOCK_SW_PREV;
        else
                rfkill->state &= ~RFKILL_BLOCK_SW_PREV;

        if (blocked)
                rfkill->state |= RFKILL_BLOCK_SW;
        else
                rfkill->state &= ~RFKILL_BLOCK_SW;

        rfkill->state |= RFKILL_BLOCK_SW_SETCALL;
        spin_unlock_irqrestore(&rfkill->lock, flags);

        err = rfkill->ops->set_block(rfkill->data, blocked);

        spin_lock_irqsave(&rfkill->lock, flags);
        if (err) {
                /*
                 * Failed -- reset status to _PREV, which may be different
                 * from what we have set _PREV to earlier in this function
                 * if rfkill_set_sw_state was invoked.
                 */
                if (rfkill->state & RFKILL_BLOCK_SW_PREV)
                        rfkill->state |= RFKILL_BLOCK_SW;
                else
                        rfkill->state &= ~RFKILL_BLOCK_SW;
        }
        rfkill->state &= ~RFKILL_BLOCK_SW_SETCALL;
        rfkill->state &= ~RFKILL_BLOCK_SW_PREV;
        curr = rfkill->state & RFKILL_BLOCK_SW;
        spin_unlock_irqrestore(&rfkill->lock, flags);

        rfkill_led_trigger_event(rfkill);
        rfkill_global_led_trigger_event();

        if (prev != curr)
                rfkill_event(rfkill);
}

static void rfkill_sync(struct rfkill *rfkill)
{
        lockdep_assert_held(&rfkill_global_mutex);

        if (!rfkill->need_sync)
                return;

        rfkill_set_block(rfkill, rfkill_global_states[rfkill->type].cur);
        rfkill->need_sync = false;
}

static void rfkill_update_global_state(enum rfkill_type type, bool blocked)
{
        int i;

        if (type != RFKILL_TYPE_ALL) {
                rfkill_global_states[type].cur = blocked;
                return;
        }

        for (i = 0; i < NUM_RFKILL_TYPES; i++)
                rfkill_global_states[i].cur = blocked;
}

#ifdef CONFIG_RFKILL_INPUT
static atomic_t rfkill_input_disabled = ATOMIC_INIT(0);

/**
 * __rfkill_switch_all - Toggle state of all switches of given type
 * @type: type of interfaces to be affected
 * @blocked: the new state
 *
 * This function sets the state of all switches of given type,
 * unless a specific switch is suspended.
 *
 * Caller must have acquired rfkill_global_mutex.
 */
static void __rfkill_switch_all(const enum rfkill_type type, bool blocked)
{
        struct rfkill *rfkill;

        rfkill_update_global_state(type, blocked);
        list_for_each_entry(rfkill, &rfkill_list, node) {
                if (rfkill->type != type && type != RFKILL_TYPE_ALL)
                        continue;

                rfkill_set_block(rfkill, blocked);
        }
}

/**
 * rfkill_switch_all - Toggle state of all switches of given type
 * @type: type of interfaces to be affected
 * @blocked: the new state
 *
 * Acquires rfkill_global_mutex and calls __rfkill_switch_all(@type, @state).
 * Please refer to __rfkill_switch_all() for details.
 *
 * Does nothing if the EPO lock is active.
 */
void rfkill_switch_all(enum rfkill_type type, bool blocked)
{
        if (atomic_read(&rfkill_input_disabled))
                return;

        mutex_lock(&rfkill_global_mutex);

        if (!rfkill_epo_lock_active)
                __rfkill_switch_all(type, blocked);

        mutex_unlock(&rfkill_global_mutex);
}

/**
 * rfkill_epo - emergency power off all transmitters
 *
 * This kicks all non-suspended rfkill devices to RFKILL_STATE_SOFT_BLOCKED,
 * ignoring everything in its path but rfkill_global_mutex and rfkill->mutex.
 *
 * The global state before the EPO is saved and can be restored later
 * using rfkill_restore_states().
 */
void rfkill_epo(void)
{
        struct rfkill *rfkill;
        int i;

        if (atomic_read(&rfkill_input_disabled))
                return;

        mutex_lock(&rfkill_global_mutex);

        rfkill_epo_lock_active = true;
        list_for_each_entry(rfkill, &rfkill_list, node)
                rfkill_set_block(rfkill, true);

        for (i = 0; i < NUM_RFKILL_TYPES; i++) {
                rfkill_global_states[i].sav = rfkill_global_states[i].cur;
                rfkill_global_states[i].cur = true;
        }

        mutex_unlock(&rfkill_global_mutex);
}

/**
 * rfkill_restore_states - restore global states
 *
 * Restore (and sync switches to) the global state from the
 * states in rfkill_default_states.  This can undo the effects of
 * a call to rfkill_epo().
 */
void rfkill_restore_states(void)
{
        int i;

        if (atomic_read(&rfkill_input_disabled))
                return;

        mutex_lock(&rfkill_global_mutex);

        rfkill_epo_lock_active = false;
        for (i = 0; i < NUM_RFKILL_TYPES; i++)
                __rfkill_switch_all(i, rfkill_global_states[i].sav);
        mutex_unlock(&rfkill_global_mutex);
}

/**
 * rfkill_remove_epo_lock - unlock state changes
 *
 * Used by rfkill-input manually unlock state changes, when
 * the EPO switch is deactivated.
 */
void rfkill_remove_epo_lock(void)
{
        if (atomic_read(&rfkill_input_disabled))
                return;

        mutex_lock(&rfkill_global_mutex);
        rfkill_epo_lock_active = false;
        mutex_unlock(&rfkill_global_mutex);
}

/**
 * rfkill_is_epo_lock_active - returns true EPO is active
 *
 * Returns 0 (false) if there is NOT an active EPO condition,
 * and 1 (true) if there is an active EPO condition, which
 * locks all radios in one of the BLOCKED states.
 *
 * Can be called in atomic context.
 */
bool rfkill_is_epo_lock_active(void)
{
        return rfkill_epo_lock_active;
}

/**
 * rfkill_get_global_sw_state - returns global state for a type
 * @type: the type to get the global state of
 *
 * Returns the current global state for a given wireless
 * device type.
 */
bool rfkill_get_global_sw_state(const enum rfkill_type type)
{
        return rfkill_global_states[type].cur;
}
#endif

bool rfkill_set_hw_state_reason(struct rfkill *rfkill,
                                bool blocked,
                                enum rfkill_hard_block_reasons reason)
{
        unsigned long flags;
        bool ret, prev;

        BUG_ON(!rfkill);

        spin_lock_irqsave(&rfkill->lock, flags);
        prev = !!(rfkill->hard_block_reasons & reason);
        if (blocked) {
                rfkill->state |= RFKILL_BLOCK_HW;
                rfkill->hard_block_reasons |= reason;
        } else {
                rfkill->hard_block_reasons &= ~reason;
                if (!rfkill->hard_block_reasons)
                        rfkill->state &= ~RFKILL_BLOCK_HW;
        }
        ret = !!(rfkill->state & RFKILL_BLOCK_ANY);
        spin_unlock_irqrestore(&rfkill->lock, flags);

        rfkill_led_trigger_event(rfkill);
        rfkill_global_led_trigger_event();

        if (rfkill->registered && prev != blocked)
                schedule_work(&rfkill->uevent_work);

        return ret;
}
EXPORT_SYMBOL(rfkill_set_hw_state_reason);

static void __rfkill_set_sw_state(struct rfkill *rfkill, bool blocked)
{
        u32 bit = RFKILL_BLOCK_SW;

        /* if in a ops->set_block right now, use other bit */
        if (rfkill->state & RFKILL_BLOCK_SW_SETCALL)
                bit = RFKILL_BLOCK_SW_PREV;

        if (blocked)
                rfkill->state |= bit;
        else
                rfkill->state &= ~bit;
}

bool rfkill_set_sw_state(struct rfkill *rfkill, bool blocked)
{
        unsigned long flags;
        bool prev, hwblock;

        BUG_ON(!rfkill);

        spin_lock_irqsave(&rfkill->lock, flags);
        prev = !!(rfkill->state & RFKILL_BLOCK_SW);
        __rfkill_set_sw_state(rfkill, blocked);
        hwblock = !!(rfkill->state & RFKILL_BLOCK_HW);
        blocked = blocked || hwblock;
        spin_unlock_irqrestore(&rfkill->lock, flags);

        if (!rfkill->registered)
                return blocked;

        if (prev != blocked && !hwblock)
                schedule_work(&rfkill->uevent_work);

        rfkill_led_trigger_event(rfkill);
        rfkill_global_led_trigger_event();

        return blocked;
}
EXPORT_SYMBOL(rfkill_set_sw_state);

void rfkill_init_sw_state(struct rfkill *rfkill, bool blocked)
{
        unsigned long flags;

        BUG_ON(!rfkill);
        BUG_ON(rfkill->registered);

        spin_lock_irqsave(&rfkill->lock, flags);
        __rfkill_set_sw_state(rfkill, blocked);
        rfkill->persistent = true;
        spin_unlock_irqrestore(&rfkill->lock, flags);
}
EXPORT_SYMBOL(rfkill_init_sw_state);

void rfkill_set_states(struct rfkill *rfkill, bool sw, bool hw)
{
        unsigned long flags;
        bool swprev, hwprev;

        BUG_ON(!rfkill);

        spin_lock_irqsave(&rfkill->lock, flags);

        /*
         * No need to care about prev/setblock ... this is for uevent only
         * and that will get triggered by rfkill_set_block anyway.
         */
        swprev = !!(rfkill->state & RFKILL_BLOCK_SW);
        hwprev = !!(rfkill->state & RFKILL_BLOCK_HW);
        __rfkill_set_sw_state(rfkill, sw);
        if (hw)
                rfkill->state |= RFKILL_BLOCK_HW;
        else
                rfkill->state &= ~RFKILL_BLOCK_HW;

        spin_unlock_irqrestore(&rfkill->lock, flags);

        if (!rfkill->registered) {
                rfkill->persistent = true;
        } else {
                if (swprev != sw || hwprev != hw)
                        schedule_work(&rfkill->uevent_work);

                rfkill_led_trigger_event(rfkill);
                rfkill_global_led_trigger_event();
        }
}
EXPORT_SYMBOL(rfkill_set_states);

static const char * const rfkill_types[] = {
        NULL, /* RFKILL_TYPE_ALL */
        "wlan",
        "bluetooth",
        "ultrawideband",
        "wimax",
        "wwan",
        "gps",
        "fm",
        "nfc",
};

enum rfkill_type rfkill_find_type(const char *name)
{
        int i;

        BUILD_BUG_ON(ARRAY_SIZE(rfkill_types) != NUM_RFKILL_TYPES);

        if (!name)
                return RFKILL_TYPE_ALL;

        for (i = 1; i < NUM_RFKILL_TYPES; i++)
                if (!strcmp(name, rfkill_types[i]))
                        return i;
        return RFKILL_TYPE_ALL;
}
EXPORT_SYMBOL(rfkill_find_type);

static ssize_t name_show(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct rfkill *rfkill = to_rfkill(dev);

        return sysfs_emit(buf, "%s\n", rfkill->name);
}
static DEVICE_ATTR_RO(name);

static ssize_t type_show(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct rfkill *rfkill = to_rfkill(dev);

        return sysfs_emit(buf, "%s\n", rfkill_types[rfkill->type]);
}
static DEVICE_ATTR_RO(type);

static ssize_t index_show(struct device *dev, struct device_attribute *attr,
                          char *buf)
{
        struct rfkill *rfkill = to_rfkill(dev);

        return sysfs_emit(buf, "%d\n", rfkill->idx);
}
static DEVICE_ATTR_RO(index);

static ssize_t persistent_show(struct device *dev,
                               struct device_attribute *attr, char *buf)
{
        struct rfkill *rfkill = to_rfkill(dev);

        return sysfs_emit(buf, "%d\n", rfkill->persistent);
}
static DEVICE_ATTR_RO(persistent);

static ssize_t hard_show(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct rfkill *rfkill = to_rfkill(dev);

        return sysfs_emit(buf, "%d\n", (rfkill->state & RFKILL_BLOCK_HW) ? 1 : 0);
}
static DEVICE_ATTR_RO(hard);

static ssize_t soft_show(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct rfkill *rfkill = to_rfkill(dev);

        mutex_lock(&rfkill_global_mutex);
        rfkill_sync(rfkill);
        mutex_unlock(&rfkill_global_mutex);

        return sysfs_emit(buf, "%d\n", (rfkill->state & RFKILL_BLOCK_SW) ? 1 : 0);
}

static ssize_t soft_store(struct device *dev, struct device_attribute *attr,
                          const char *buf, size_t count)
{
        struct rfkill *rfkill = to_rfkill(dev);
        unsigned long state;
        int err;

        if (!capable(CAP_NET_ADMIN))
                return -EPERM;

        err = kstrtoul(buf, 0, &state);
        if (err)
                return err;

        if (state > 1 )
                return -EINVAL;

        mutex_lock(&rfkill_global_mutex);
        rfkill_sync(rfkill);
        rfkill_set_block(rfkill, state);
        mutex_unlock(&rfkill_global_mutex);

        return count;
}
static DEVICE_ATTR_RW(soft);

static ssize_t hard_block_reasons_show(struct device *dev,
                                       struct device_attribute *attr,
                                       char *buf)
{
        struct rfkill *rfkill = to_rfkill(dev);

        return sysfs_emit(buf, "0x%lx\n", rfkill->hard_block_reasons);
}
static DEVICE_ATTR_RO(hard_block_reasons);

static u8 user_state_from_blocked(unsigned long state)
{
        if (state & RFKILL_BLOCK_HW)
                return RFKILL_USER_STATE_HARD_BLOCKED;
        if (state & RFKILL_BLOCK_SW)
                return RFKILL_USER_STATE_SOFT_BLOCKED;

        return RFKILL_USER_STATE_UNBLOCKED;
}

static ssize_t state_show(struct device *dev, struct device_attribute *attr,
                          char *buf)
{
        struct rfkill *rfkill = to_rfkill(dev);

        mutex_lock(&rfkill_global_mutex);
        rfkill_sync(rfkill);
        mutex_unlock(&rfkill_global_mutex);

        return sysfs_emit(buf, "%d\n", user_state_from_blocked(rfkill->state));
}

static ssize_t state_store(struct device *dev, struct device_attribute *attr,
                           const char *buf, size_t count)
{
        struct rfkill *rfkill = to_rfkill(dev);
        unsigned long state;
        int err;

        if (!capable(CAP_NET_ADMIN))
                return -EPERM;

        err = kstrtoul(buf, 0, &state);
        if (err)
                return err;

        if (state != RFKILL_USER_STATE_SOFT_BLOCKED &&
            state != RFKILL_USER_STATE_UNBLOCKED)
                return -EINVAL;

        mutex_lock(&rfkill_global_mutex);
        rfkill_sync(rfkill);
        rfkill_set_block(rfkill, state == RFKILL_USER_STATE_SOFT_BLOCKED);
        mutex_unlock(&rfkill_global_mutex);

        return count;
}
static DEVICE_ATTR_RW(state);

static struct attribute *rfkill_dev_attrs[] = {
        &dev_attr_name.attr,
        &dev_attr_type.attr,
        &dev_attr_index.attr,
        &dev_attr_persistent.attr,
        &dev_attr_state.attr,
        &dev_attr_soft.attr,
        &dev_attr_hard.attr,
        &dev_attr_hard_block_reasons.attr,
        NULL,
};
ATTRIBUTE_GROUPS(rfkill_dev);

static void rfkill_release(struct device *dev)
{
        struct rfkill *rfkill = to_rfkill(dev);

        kfree(rfkill);
}

static int rfkill_dev_uevent(const struct device *dev, struct kobj_uevent_env *env)
{
        struct rfkill *rfkill = to_rfkill(dev);
        unsigned long flags;
        unsigned long reasons;
        u32 state;
        int error;

        error = add_uevent_var(env, "RFKILL_NAME=%s", rfkill->name);
        if (error)
                return error;
        error = add_uevent_var(env, "RFKILL_TYPE=%s",
                               rfkill_types[rfkill->type]);
        if (error)
                return error;
        spin_lock_irqsave(&rfkill->lock, flags);
        state = rfkill->state;
        reasons = rfkill->hard_block_reasons;
        spin_unlock_irqrestore(&rfkill->lock, flags);
        error = add_uevent_var(env, "RFKILL_STATE=%d",
                               user_state_from_blocked(state));
        if (error)
                return error;
        return add_uevent_var(env, "RFKILL_HW_BLOCK_REASON=0x%lx", reasons);
}

void rfkill_pause_polling(struct rfkill *rfkill)
{
        BUG_ON(!rfkill);

        if (!rfkill->ops->poll)
                return;

        rfkill->polling_paused = true;
        cancel_delayed_work_sync(&rfkill->poll_work);
}
EXPORT_SYMBOL(rfkill_pause_polling);

void rfkill_resume_polling(struct rfkill *rfkill)
{
        BUG_ON(!rfkill);

        if (!rfkill->ops->poll)
                return;

        rfkill->polling_paused = false;

        if (rfkill->suspended)
                return;

        queue_delayed_work(system_power_efficient_wq,
                           &rfkill->poll_work, 0);
}
EXPORT_SYMBOL(rfkill_resume_polling);

#ifdef CONFIG_PM_SLEEP
static int rfkill_suspend(struct device *dev)
{
        struct rfkill *rfkill = to_rfkill(dev);

        rfkill->suspended = true;
        cancel_delayed_work_sync(&rfkill->poll_work);

        return 0;
}

static int rfkill_resume(struct device *dev)
{
        struct rfkill *rfkill = to_rfkill(dev);
        bool cur;

        rfkill->suspended = false;

        if (!rfkill->registered)
                return 0;

        if (!rfkill->persistent) {
                cur = !!(rfkill->state & RFKILL_BLOCK_SW);
                rfkill_set_block(rfkill, cur);
        }

        if (rfkill->ops->poll && !rfkill->polling_paused)
                queue_delayed_work(system_power_efficient_wq,
                                   &rfkill->poll_work, 0);

        return 0;
}

static SIMPLE_DEV_PM_OPS(rfkill_pm_ops, rfkill_suspend, rfkill_resume);
#define RFKILL_PM_OPS (&rfkill_pm_ops)
#else
#define RFKILL_PM_OPS NULL
#endif

static struct class rfkill_class = {
        .name                = "rfkill",
        .dev_release        = rfkill_release,
        .dev_groups        = rfkill_dev_groups,
        .dev_uevent        = rfkill_dev_uevent,
        .pm                = RFKILL_PM_OPS,
};

bool rfkill_blocked(struct rfkill *rfkill)
{
        unsigned long flags;
        u32 state;

        spin_lock_irqsave(&rfkill->lock, flags);
        state = rfkill->state;
        spin_unlock_irqrestore(&rfkill->lock, flags);

        return !!(state & RFKILL_BLOCK_ANY);
}
EXPORT_SYMBOL(rfkill_blocked);

bool rfkill_soft_blocked(struct rfkill *rfkill)
{
        unsigned long flags;
        u32 state;

        spin_lock_irqsave(&rfkill->lock, flags);
        state = rfkill->state;
        spin_unlock_irqrestore(&rfkill->lock, flags);

        return !!(state & RFKILL_BLOCK_SW);
}
EXPORT_SYMBOL(rfkill_soft_blocked);

struct rfkill * __must_check rfkill_alloc(const char *name,
                                          struct device *parent,
                                          const enum rfkill_type type,
                                          const struct rfkill_ops *ops,
                                          void *ops_data)
{
        struct rfkill *rfkill;
        struct device *dev;

        if (WARN_ON(!ops))
                return NULL;

        if (WARN_ON(!ops->set_block))
                return NULL;

        if (WARN_ON(!name))
                return NULL;

        if (WARN_ON(type == RFKILL_TYPE_ALL || type >= NUM_RFKILL_TYPES))
                return NULL;

        rfkill = kzalloc(sizeof(*rfkill) + strlen(name) + 1, GFP_KERNEL);
        if (!rfkill)
                return NULL;

        spin_lock_init(&rfkill->lock);
        INIT_LIST_HEAD(&rfkill->node);
        rfkill->type = type;
        strcpy(rfkill->name, name);
        rfkill->ops = ops;
        rfkill->data = ops_data;

        dev = &rfkill->dev;
        dev->class = &rfkill_class;
        dev->parent = parent;
        device_initialize(dev);

        return rfkill;
}
EXPORT_SYMBOL(rfkill_alloc);

static void rfkill_poll(struct work_struct *work)
{
        struct rfkill *rfkill;

        rfkill = container_of(work, struct rfkill, poll_work.work);

        /*
         * Poll hardware state -- driver will use one of the
         * rfkill_set{,_hw,_sw}_state functions and use its
         * return value to update the current status.
         */
        rfkill->ops->poll(rfkill, rfkill->data);

        queue_delayed_work(system_power_efficient_wq,
                &rfkill->poll_work,
                round_jiffies_relative(POLL_INTERVAL));
}

static void rfkill_uevent_work(struct work_struct *work)
{
        struct rfkill *rfkill;

        rfkill = container_of(work, struct rfkill, uevent_work);

        mutex_lock(&rfkill_global_mutex);
        rfkill_event(rfkill);
        mutex_unlock(&rfkill_global_mutex);
}

static void rfkill_sync_work(struct work_struct *work)
{
        struct rfkill *rfkill = container_of(work, struct rfkill, sync_work);

        mutex_lock(&rfkill_global_mutex);
        rfkill_sync(rfkill);
        mutex_unlock(&rfkill_global_mutex);
}

int __must_check rfkill_register(struct rfkill *rfkill)
{
        static unsigned long rfkill_no;
        struct device *dev;
        int error;

        if (!rfkill)
                return -EINVAL;

        dev = &rfkill->dev;

        mutex_lock(&rfkill_global_mutex);

        if (rfkill->registered) {
                error = -EALREADY;
                goto unlock;
        }

        rfkill->idx = rfkill_no;
        dev_set_name(dev, "rfkill%lu", rfkill_no);
        rfkill_no++;

        list_add_tail(&rfkill->node, &rfkill_list);

        error = device_add(dev);
        if (error)
                goto remove;

        error = rfkill_led_trigger_register(rfkill);
        if (error)
                goto devdel;

        rfkill->registered = true;

        INIT_DELAYED_WORK(&rfkill->poll_work, rfkill_poll);
        INIT_WORK(&rfkill->uevent_work, rfkill_uevent_work);
        INIT_WORK(&rfkill->sync_work, rfkill_sync_work);

        if (rfkill->ops->poll)
                queue_delayed_work(system_power_efficient_wq,
                        &rfkill->poll_work,
                        round_jiffies_relative(POLL_INTERVAL));

        if (!rfkill->persistent || rfkill_epo_lock_active) {
                rfkill->need_sync = true;
                schedule_work(&rfkill->sync_work);
        } else {
#ifdef CONFIG_RFKILL_INPUT
                bool soft_blocked = !!(rfkill->state & RFKILL_BLOCK_SW);

                if (!atomic_read(&rfkill_input_disabled))
                        __rfkill_switch_all(rfkill->type, soft_blocked);
#endif
        }

        rfkill_global_led_trigger_event();
        rfkill_send_events(rfkill, RFKILL_OP_ADD);

        mutex_unlock(&rfkill_global_mutex);
        return 0;

 devdel:
        device_del(&rfkill->dev);
 remove:
        list_del_init(&rfkill->node);
 unlock:
        mutex_unlock(&rfkill_global_mutex);
        return error;
}
EXPORT_SYMBOL(rfkill_register);

void rfkill_unregister(struct rfkill *rfkill)
{
        BUG_ON(!rfkill);

        if (rfkill->ops->poll)
                cancel_delayed_work_sync(&rfkill->poll_work);

        cancel_work_sync(&rfkill->uevent_work);
        cancel_work_sync(&rfkill->sync_work);

        rfkill->registered = false;

        device_del(&rfkill->dev);

        mutex_lock(&rfkill_global_mutex);
        rfkill_send_events(rfkill, RFKILL_OP_DEL);
        list_del_init(&rfkill->node);
        rfkill_global_led_trigger_event();
        mutex_unlock(&rfkill_global_mutex);

        rfkill_led_trigger_unregister(rfkill);
}
EXPORT_SYMBOL(rfkill_unregister);

void rfkill_destroy(struct rfkill *rfkill)
{
        if (rfkill)
                put_device(&rfkill->dev);
}
EXPORT_SYMBOL(rfkill_destroy);

static int rfkill_fop_open(struct inode *inode, struct file *file)
{
        struct rfkill_data *data;
        struct rfkill *rfkill;
        struct rfkill_int_event *ev, *tmp;

        data = kzalloc(sizeof(*data), GFP_KERNEL);
        if (!data)
                return -ENOMEM;

        data->max_size = RFKILL_EVENT_SIZE_V1;

        INIT_LIST_HEAD(&data->events);
        mutex_init(&data->mtx);
        init_waitqueue_head(&data->read_wait);

        mutex_lock(&rfkill_global_mutex);
        /*
         * start getting events from elsewhere but hold mtx to get
         * startup events added first
         */

        list_for_each_entry(rfkill, &rfkill_list, node) {
                ev = kzalloc(sizeof(*ev), GFP_KERNEL);
                if (!ev)
                        goto free;
                rfkill_sync(rfkill);
                rfkill_fill_event(&ev->ev, rfkill, RFKILL_OP_ADD);
                mutex_lock(&data->mtx);
                list_add_tail(&ev->list, &data->events);
                mutex_unlock(&data->mtx);
        }
        list_add(&data->list, &rfkill_fds);
        mutex_unlock(&rfkill_global_mutex);

        file->private_data = data;

        return stream_open(inode, file);

 free:
        mutex_unlock(&rfkill_global_mutex);
        mutex_destroy(&data->mtx);
        list_for_each_entry_safe(ev, tmp, &data->events, list)
                kfree(ev);
        kfree(data);
        return -ENOMEM;
}

static __poll_t rfkill_fop_poll(struct file *file, poll_table *wait)
{
        struct rfkill_data *data = file->private_data;
        __poll_t res = EPOLLOUT | EPOLLWRNORM;

        poll_wait(file, &data->read_wait, wait);

        mutex_lock(&data->mtx);
        if (!list_empty(&data->events))
                res = EPOLLIN | EPOLLRDNORM;
        mutex_unlock(&data->mtx);

        return res;
}

static ssize_t rfkill_fop_read(struct file *file, char __user *buf,
                               size_t count, loff_t *pos)
{
        struct rfkill_data *data = file->private_data;
        struct rfkill_int_event *ev;
        unsigned long sz;
        int ret;

        mutex_lock(&data->mtx);

        while (list_empty(&data->events)) {
                if (file->f_flags & O_NONBLOCK) {
                        ret = -EAGAIN;
                        goto out;
                }
                mutex_unlock(&data->mtx);
                /* since we re-check and it just compares pointers,
                 * using !list_empty() without locking isn't a problem
                 */
                ret = wait_event_interruptible(data->read_wait,
                                               !list_empty(&data->events));
                mutex_lock(&data->mtx);

                if (ret)
                        goto out;
        }

        ev = list_first_entry(&data->events, struct rfkill_int_event,
                                list);

        sz = min_t(unsigned long, sizeof(ev->ev), count);
        sz = min_t(unsigned long, sz, data->max_size);
        ret = sz;
        if (copy_to_user(buf, &ev->ev, sz))
                ret = -EFAULT;

        list_del(&ev->list);
        kfree(ev);
 out:
        mutex_unlock(&data->mtx);
        return ret;
}

static ssize_t rfkill_fop_write(struct file *file, const char __user *buf,
                                size_t count, loff_t *pos)
{
        struct rfkill_data *data = file->private_data;
        struct rfkill *rfkill;
        struct rfkill_event_ext ev;
        int ret;

        /* we don't need the 'hard' variable but accept it */
        if (count < RFKILL_EVENT_SIZE_V1 - 1)
                return -EINVAL;

        /*
         * Copy as much data as we can accept into our 'ev' buffer,
         * but tell userspace how much we've copied so it can determine
         * our API version even in a write() call, if it cares.
         */
        count = min(count, sizeof(ev));
        count = min_t(size_t, count, data->max_size);
        if (copy_from_user(&ev, buf, count))
                return -EFAULT;

        if (ev.type >= NUM_RFKILL_TYPES)
                return -EINVAL;

        mutex_lock(&rfkill_global_mutex);

        switch (ev.op) {
        case RFKILL_OP_CHANGE_ALL:
                rfkill_update_global_state(ev.type, ev.soft);
                list_for_each_entry(rfkill, &rfkill_list, node)
                        if (rfkill->type == ev.type ||
                            ev.type == RFKILL_TYPE_ALL)
                                rfkill_set_block(rfkill, ev.soft);
                ret = 0;
                break;
        case RFKILL_OP_CHANGE:
                list_for_each_entry(rfkill, &rfkill_list, node)
                        if (rfkill->idx == ev.idx &&
                            (rfkill->type == ev.type ||
                             ev.type == RFKILL_TYPE_ALL))
                                rfkill_set_block(rfkill, ev.soft);
                ret = 0;
                break;
        default:
                ret = -EINVAL;
                break;
        }

        mutex_unlock(&rfkill_global_mutex);

        return ret ?: count;
}

static int rfkill_fop_release(struct inode *inode, struct file *file)
{
        struct rfkill_data *data = file->private_data;
        struct rfkill_int_event *ev, *tmp;

        mutex_lock(&rfkill_global_mutex);
        list_del(&data->list);
        mutex_unlock(&rfkill_global_mutex);

        mutex_destroy(&data->mtx);
        list_for_each_entry_safe(ev, tmp, &data->events, list)
                kfree(ev);

#ifdef CONFIG_RFKILL_INPUT
        if (data->input_handler)
                if (atomic_dec_return(&rfkill_input_disabled) == 0)
                        printk(KERN_DEBUG "rfkill: input handler enabled\n");
#endif

        kfree(data);

        return 0;
}

static long rfkill_fop_ioctl(struct file *file, unsigned int cmd,
                             unsigned long arg)
{
        struct rfkill_data *data = file->private_data;
        int ret = -ENOTTY;
        u32 size;

        if (_IOC_TYPE(cmd) != RFKILL_IOC_MAGIC)
                return -ENOTTY;

        mutex_lock(&data->mtx);
        switch (_IOC_NR(cmd)) {
#ifdef CONFIG_RFKILL_INPUT
        case RFKILL_IOC_NOINPUT:
                if (!data->input_handler) {
                        if (atomic_inc_return(&rfkill_input_disabled) == 1)
                                printk(KERN_DEBUG "rfkill: input handler disabled\n");
                        data->input_handler = true;
                }
                ret = 0;
                break;
#endif
        case RFKILL_IOC_MAX_SIZE:
                if (get_user(size, (__u32 __user *)arg)) {
                        ret = -EFAULT;
                        break;
                }
                if (size < RFKILL_EVENT_SIZE_V1 || size > U8_MAX) {
                        ret = -EINVAL;
                        break;
                }
                data->max_size = size;
                ret = 0;
                break;
        default:
                break;
        }
        mutex_unlock(&data->mtx);

        return ret;
}

static const struct file_operations rfkill_fops = {
        .owner                = THIS_MODULE,
        .open                = rfkill_fop_open,
        .read                = rfkill_fop_read,
        .write                = rfkill_fop_write,
        .poll                = rfkill_fop_poll,
        .release        = rfkill_fop_release,
        .unlocked_ioctl        = rfkill_fop_ioctl,
        .compat_ioctl        = compat_ptr_ioctl,
};

#define RFKILL_NAME "rfkill"

static struct miscdevice rfkill_miscdev = {
        .fops        = &rfkill_fops,
        .name        = RFKILL_NAME,
        .minor        = RFKILL_MINOR,
};

static int __init rfkill_init(void)
{
        int error;

        rfkill_update_global_state(RFKILL_TYPE_ALL, !rfkill_default_state);

        error = class_register(&rfkill_class);
        if (error)
                goto error_class;

        error = misc_register(&rfkill_miscdev);
        if (error)
                goto error_misc;

        error = rfkill_global_led_trigger_register();
        if (error)
                goto error_led_trigger;

#ifdef CONFIG_RFKILL_INPUT
        error = rfkill_handler_init();
        if (error)
                goto error_input;
#endif

        return 0;

#ifdef CONFIG_RFKILL_INPUT
error_input:
        rfkill_global_led_trigger_unregister();
#endif
error_led_trigger:
        misc_deregister(&rfkill_miscdev);
error_misc:
        class_unregister(&rfkill_class);
error_class:
        return error;
}
subsys_initcall(rfkill_init);

static void __exit rfkill_exit(void)
{
#ifdef CONFIG_RFKILL_INPUT
        rfkill_handler_exit();
#endif
        rfkill_global_led_trigger_unregister();
        misc_deregister(&rfkill_miscdev);
        class_unregister(&rfkill_class);
}
module_exit(rfkill_exit);

MODULE_ALIAS_MISCDEV(RFKILL_MINOR);
MODULE_ALIAS("devname:" RFKILL_NAME);






































































 5793 

 5825 





























  198 

  436 





  147 










   31 























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * fs/kernfs/kernfs-internal.h - kernfs internal header file
 *
 * Copyright (c) 2001-3 Patrick Mochel
 * Copyright (c) 2007 SUSE Linux Products GmbH
 * Copyright (c) 2007, 2013 Tejun Heo <teheo@suse.de>
 */

#ifndef __KERNFS_INTERNAL_H
#define __KERNFS_INTERNAL_H

#include <linux/lockdep.h>
#include <linux/fs.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/xattr.h>

#include <linux/kernfs.h>
#include <linux/fs_context.h>

struct kernfs_iattrs {
        kuid_t                        ia_uid;
        kgid_t                        ia_gid;
        struct timespec64        ia_atime;
        struct timespec64        ia_mtime;
        struct timespec64        ia_ctime;

        struct simple_xattrs        xattrs;
        atomic_t                nr_user_xattrs;
        atomic_t                user_xattr_size;
};

struct kernfs_root {
        /* published fields */
        struct kernfs_node        *kn;
        unsigned int                flags;        /* KERNFS_ROOT_* flags */

        /* private fields, do not use outside kernfs proper */
        struct idr                ino_idr;
        u32                        last_id_lowbits;
        u32                        id_highbits;
        struct kernfs_syscall_ops *syscall_ops;

        /* list of kernfs_super_info of this root, protected by kernfs_rwsem */
        struct list_head        supers;

        wait_queue_head_t        deactivate_waitq;
        struct rw_semaphore        kernfs_rwsem;
        struct rw_semaphore        kernfs_iattr_rwsem;
        struct rw_semaphore        kernfs_supers_rwsem;

        struct rcu_head                rcu;
};

/* +1 to avoid triggering overflow warning when negating it */
#define KN_DEACTIVATED_BIAS                (INT_MIN + 1)

/* KERNFS_TYPE_MASK and types are defined in include/linux/kernfs.h */

/**
 * kernfs_root - find out the kernfs_root a kernfs_node belongs to
 * @kn: kernfs_node of interest
 *
 * Return: the kernfs_root @kn belongs to.
 */
static inline struct kernfs_root *kernfs_root(struct kernfs_node *kn)
{
        /* if parent exists, it's always a dir; otherwise, @sd is a dir */
        if (kn->parent)
                kn = kn->parent;
        return kn->dir.root;
}

/*
 * mount.c
 */
struct kernfs_super_info {
        struct super_block        *sb;

        /*
         * The root associated with this super_block.  Each super_block is
         * identified by the root and ns it's associated with.
         */
        struct kernfs_root        *root;

        /*
         * Each sb is associated with one namespace tag, currently the
         * network namespace of the task which mounted this kernfs
         * instance.  If multiple tags become necessary, make the following
         * an array and compare kernfs_node tag against every entry.
         */
        const void                *ns;

        /* anchored at kernfs_root->supers, protected by kernfs_rwsem */
        struct list_head        node;
};
#define kernfs_info(SB) ((struct kernfs_super_info *)(SB->s_fs_info))

static inline struct kernfs_node *kernfs_dentry_node(struct dentry *dentry)
{
        if (d_really_is_negative(dentry))
                return NULL;
        return d_inode(dentry)->i_private;
}

static inline void kernfs_set_rev(struct kernfs_node *parent,
                                  struct dentry *dentry)
{
        dentry->d_time = parent->dir.rev;
}

static inline void kernfs_inc_rev(struct kernfs_node *parent)
{
        parent->dir.rev++;
}

static inline bool kernfs_dir_changed(struct kernfs_node *parent,
                                      struct dentry *dentry)
{
        if (parent->dir.rev != dentry->d_time)
                return true;
        return false;
}

extern const struct super_operations kernfs_sops;
extern struct kmem_cache *kernfs_node_cache, *kernfs_iattrs_cache;

/*
 * inode.c
 */
extern const struct xattr_handler * const kernfs_xattr_handlers[];
void kernfs_evict_inode(struct inode *inode);
int kernfs_iop_permission(struct mnt_idmap *idmap,
                          struct inode *inode, int mask);
int kernfs_iop_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
                       struct iattr *iattr);
int kernfs_iop_getattr(struct mnt_idmap *idmap,
                       const struct path *path, struct kstat *stat,
                       u32 request_mask, unsigned int query_flags);
ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size);
int __kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr);

/*
 * dir.c
 */
extern const struct dentry_operations kernfs_dops;
extern const struct file_operations kernfs_dir_fops;
extern const struct inode_operations kernfs_dir_iops;

struct kernfs_node *kernfs_get_active(struct kernfs_node *kn);
void kernfs_put_active(struct kernfs_node *kn);
int kernfs_add_one(struct kernfs_node *kn);
struct kernfs_node *kernfs_new_node(struct kernfs_node *parent,
                                    const char *name, umode_t mode,
                                    kuid_t uid, kgid_t gid,
                                    unsigned flags);

/*
 * file.c
 */
extern const struct file_operations kernfs_file_fops;

bool kernfs_should_drain_open_files(struct kernfs_node *kn);
void kernfs_drain_open_files(struct kernfs_node *kn);

/*
 * symlink.c
 */
extern const struct inode_operations kernfs_symlink_iops;

/*
 * kernfs locks
 */
extern struct kernfs_global_locks *kernfs_locks;
#endif        /* __KERNFS_INTERNAL_H */

























































































































 8773 

























































































































































































































































































































































































































































































 8777 




 8777 






































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
// SPDX-License-Identifier: GPL-2.0
/*
 * Detect hard and soft lockups on a system
 *
 * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
 *
 * Note: Most of this code is borrowed heavily from the original softlockup
 * detector, so thanks to Ingo for the initial implementation.
 * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
 * to those contributors as well.
 */

#define pr_fmt(fmt) "watchdog: " fmt

#include <linux/cpu.h>
#include <linux/init.h>
#include <linux/irq.h>
#include <linux/irqdesc.h>
#include <linux/kernel_stat.h>
#include <linux/kvm_para.h>
#include <linux/math64.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/nmi.h>
#include <linux/stop_machine.h>
#include <linux/sysctl.h>
#include <linux/tick.h>

#include <linux/sched/clock.h>
#include <linux/sched/debug.h>
#include <linux/sched/isolation.h>

#include <asm/irq_regs.h>

static DEFINE_MUTEX(watchdog_mutex);

#if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HARDLOCKUP_DETECTOR_SPARC64)
# define WATCHDOG_HARDLOCKUP_DEFAULT        1
#else
# define WATCHDOG_HARDLOCKUP_DEFAULT        0
#endif

#define NUM_SAMPLE_PERIODS        5

unsigned long __read_mostly watchdog_enabled;
int __read_mostly watchdog_user_enabled = 1;
static int __read_mostly watchdog_hardlockup_user_enabled = WATCHDOG_HARDLOCKUP_DEFAULT;
static int __read_mostly watchdog_softlockup_user_enabled = 1;
int __read_mostly watchdog_thresh = 10;
static int __read_mostly watchdog_hardlockup_available;

struct cpumask watchdog_cpumask __read_mostly;
unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);

#ifdef CONFIG_HARDLOCKUP_DETECTOR

# ifdef CONFIG_SMP
int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
# endif /* CONFIG_SMP */

/*
 * Should we panic when a soft-lockup or hard-lockup occurs:
 */
unsigned int __read_mostly hardlockup_panic =
                        IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);
/*
 * We may not want to enable hard lockup detection by default in all cases,
 * for example when running the kernel as a guest on a hypervisor. In these
 * cases this function can be called to disable hard lockup detection. This
 * function should only be executed once by the boot processor before the
 * kernel command line parameters are parsed, because otherwise it is not
 * possible to override this in hardlockup_panic_setup().
 */
void __init hardlockup_detector_disable(void)
{
        watchdog_hardlockup_user_enabled = 0;
}

static int __init hardlockup_panic_setup(char *str)
{
next:
        if (!strncmp(str, "panic", 5))
                hardlockup_panic = 1;
        else if (!strncmp(str, "nopanic", 7))
                hardlockup_panic = 0;
        else if (!strncmp(str, "0", 1))
                watchdog_hardlockup_user_enabled = 0;
        else if (!strncmp(str, "1", 1))
                watchdog_hardlockup_user_enabled = 1;
        else if (!strncmp(str, "r", 1))
                hardlockup_config_perf_event(str + 1);
        while (*(str++)) {
                if (*str == ',') {
                        str++;
                        goto next;
                }
        }
        return 1;
}
__setup("nmi_watchdog=", hardlockup_panic_setup);

#endif /* CONFIG_HARDLOCKUP_DETECTOR */

#if defined(CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER)

static DEFINE_PER_CPU(atomic_t, hrtimer_interrupts);
static DEFINE_PER_CPU(int, hrtimer_interrupts_saved);
static DEFINE_PER_CPU(bool, watchdog_hardlockup_warned);
static DEFINE_PER_CPU(bool, watchdog_hardlockup_touched);
static unsigned long hard_lockup_nmi_warn;

notrace void arch_touch_nmi_watchdog(void)
{
        /*
         * Using __raw here because some code paths have
         * preemption enabled.  If preemption is enabled
         * then interrupts should be enabled too, in which
         * case we shouldn't have to worry about the watchdog
         * going off.
         */
        raw_cpu_write(watchdog_hardlockup_touched, true);
}
EXPORT_SYMBOL(arch_touch_nmi_watchdog);

void watchdog_hardlockup_touch_cpu(unsigned int cpu)
{
        per_cpu(watchdog_hardlockup_touched, cpu) = true;
}

static bool is_hardlockup(unsigned int cpu)
{
        int hrint = atomic_read(&per_cpu(hrtimer_interrupts, cpu));

        if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint)
                return true;

        /*
         * NOTE: we don't need any fancy atomic_t or READ_ONCE/WRITE_ONCE
         * for hrtimer_interrupts_saved. hrtimer_interrupts_saved is
         * written/read by a single CPU.
         */
        per_cpu(hrtimer_interrupts_saved, cpu) = hrint;

        return false;
}

static void watchdog_hardlockup_kick(void)
{
        int new_interrupts;

        new_interrupts = atomic_inc_return(this_cpu_ptr(&hrtimer_interrupts));
        watchdog_buddy_check_hardlockup(new_interrupts);
}

void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs)
{
        if (per_cpu(watchdog_hardlockup_touched, cpu)) {
                per_cpu(watchdog_hardlockup_touched, cpu) = false;
                return;
        }

        /*
         * Check for a hardlockup by making sure the CPU's timer
         * interrupt is incrementing. The timer interrupt should have
         * fired multiple times before we overflow'd. If it hasn't
         * then this is a good indication the cpu is stuck
         */
        if (is_hardlockup(cpu)) {
                unsigned int this_cpu = smp_processor_id();
                unsigned long flags;

                /* Only print hardlockups once. */
                if (per_cpu(watchdog_hardlockup_warned, cpu))
                        return;

                /*
                 * Prevent multiple hard-lockup reports if one cpu is already
                 * engaged in dumping all cpu back traces.
                 */
                if (sysctl_hardlockup_all_cpu_backtrace) {
                        if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn))
                                return;
                }

                /*
                 * NOTE: we call printk_cpu_sync_get_irqsave() after printing
                 * the lockup message. While it would be nice to serialize
                 * that printout, we really want to make sure that if some
                 * other CPU somehow locked up while holding the lock associated
                 * with printk_cpu_sync_get_irqsave() that we can still at least
                 * get the message about the lockup out.
                 */
                pr_emerg("Watchdog detected hard LOCKUP on cpu %d\n", cpu);
                printk_cpu_sync_get_irqsave(flags);

                print_modules();
                print_irqtrace_events(current);
                if (cpu == this_cpu) {
                        if (regs)
                                show_regs(regs);
                        else
                                dump_stack();
                        printk_cpu_sync_put_irqrestore(flags);
                } else {
                        printk_cpu_sync_put_irqrestore(flags);
                        trigger_single_cpu_backtrace(cpu);
                }

                if (sysctl_hardlockup_all_cpu_backtrace) {
                        trigger_allbutcpu_cpu_backtrace(cpu);
                        if (!hardlockup_panic)
                                clear_bit_unlock(0, &hard_lockup_nmi_warn);
                }

                if (hardlockup_panic)
                        nmi_panic(regs, "Hard LOCKUP");

                per_cpu(watchdog_hardlockup_warned, cpu) = true;
        } else {
                per_cpu(watchdog_hardlockup_warned, cpu) = false;
        }
}

#else /* CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */

static inline void watchdog_hardlockup_kick(void) { }

#endif /* !CONFIG_HARDLOCKUP_DETECTOR_COUNTS_HRTIMER */

/*
 * These functions can be overridden based on the configured hardlockdup detector.
 *
 * watchdog_hardlockup_enable/disable can be implemented to start and stop when
 * softlockup watchdog start and stop. The detector must select the
 * SOFTLOCKUP_DETECTOR Kconfig.
 */
void __weak watchdog_hardlockup_enable(unsigned int cpu) { }

void __weak watchdog_hardlockup_disable(unsigned int cpu) { }

/*
 * Watchdog-detector specific API.
 *
 * Return 0 when hardlockup watchdog is available, negative value otherwise.
 * Note that the negative value means that a delayed probe might
 * succeed later.
 */
int __weak __init watchdog_hardlockup_probe(void)
{
        return -ENODEV;
}

/**
 * watchdog_hardlockup_stop - Stop the watchdog for reconfiguration
 *
 * The reconfiguration steps are:
 * watchdog_hardlockup_stop();
 * update_variables();
 * watchdog_hardlockup_start();
 */
void __weak watchdog_hardlockup_stop(void) { }

/**
 * watchdog_hardlockup_start - Start the watchdog after reconfiguration
 *
 * Counterpart to watchdog_hardlockup_stop().
 *
 * The following variables have been updated in update_variables() and
 * contain the currently valid configuration:
 * - watchdog_enabled
 * - watchdog_thresh
 * - watchdog_cpumask
 */
void __weak watchdog_hardlockup_start(void) { }

/**
 * lockup_detector_update_enable - Update the sysctl enable bit
 *
 * Caller needs to make sure that the hard watchdogs are off, so this
 * can't race with watchdog_hardlockup_disable().
 */
static void lockup_detector_update_enable(void)
{
        watchdog_enabled = 0;
        if (!watchdog_user_enabled)
                return;
        if (watchdog_hardlockup_available && watchdog_hardlockup_user_enabled)
                watchdog_enabled |= WATCHDOG_HARDLOCKUP_ENABLED;
        if (watchdog_softlockup_user_enabled)
                watchdog_enabled |= WATCHDOG_SOFTOCKUP_ENABLED;
}

#ifdef CONFIG_SOFTLOCKUP_DETECTOR

/*
 * Delay the soflockup report when running a known slow code.
 * It does _not_ affect the timestamp of the last successdul reschedule.
 */
#define SOFTLOCKUP_DELAY_REPORT        ULONG_MAX

#ifdef CONFIG_SMP
int __read_mostly sysctl_softlockup_all_cpu_backtrace;
#endif

static struct cpumask watchdog_allowed_mask __read_mostly;

/* Global variables, exported for sysctl */
unsigned int __read_mostly softlockup_panic =
                        IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC);

static bool softlockup_initialized __read_mostly;
static u64 __read_mostly sample_period;

/* Timestamp taken after the last successful reschedule. */
static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
/* Timestamp of the last softlockup report. */
static DEFINE_PER_CPU(unsigned long, watchdog_report_ts);
static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
static DEFINE_PER_CPU(bool, softlockup_touch_sync);
static unsigned long soft_lockup_nmi_warn;

static int __init softlockup_panic_setup(char *str)
{
        softlockup_panic = simple_strtoul(str, NULL, 0);
        return 1;
}
__setup("softlockup_panic=", softlockup_panic_setup);

static int __init nowatchdog_setup(char *str)
{
        watchdog_user_enabled = 0;
        return 1;
}
__setup("nowatchdog", nowatchdog_setup);

static int __init nosoftlockup_setup(char *str)
{
        watchdog_softlockup_user_enabled = 0;
        return 1;
}
__setup("nosoftlockup", nosoftlockup_setup);

static int __init watchdog_thresh_setup(char *str)
{
        get_option(&str, &watchdog_thresh);
        return 1;
}
__setup("watchdog_thresh=", watchdog_thresh_setup);

static void __lockup_detector_cleanup(void);

#ifdef CONFIG_SOFTLOCKUP_DETECTOR_INTR_STORM
enum stats_per_group {
        STATS_SYSTEM,
        STATS_SOFTIRQ,
        STATS_HARDIRQ,
        STATS_IDLE,
        NUM_STATS_PER_GROUP,
};

static const enum cpu_usage_stat tracked_stats[NUM_STATS_PER_GROUP] = {
        CPUTIME_SYSTEM,
        CPUTIME_SOFTIRQ,
        CPUTIME_IRQ,
        CPUTIME_IDLE,
};

static DEFINE_PER_CPU(u16, cpustat_old[NUM_STATS_PER_GROUP]);
static DEFINE_PER_CPU(u8, cpustat_util[NUM_SAMPLE_PERIODS][NUM_STATS_PER_GROUP]);
static DEFINE_PER_CPU(u8, cpustat_tail);

/*
 * We don't need nanosecond resolution. A granularity of 16ms is
 * sufficient for our precision, allowing us to use u16 to store
 * cpustats, which will roll over roughly every ~1000 seconds.
 * 2^24 ~= 16 * 10^6
 */
static u16 get_16bit_precision(u64 data_ns)
{
        return data_ns >> 24LL; /* 2^24ns ~= 16.8ms */
}

static void update_cpustat(void)
{
        int i;
        u8 util;
        u16 old_stat, new_stat;
        struct kernel_cpustat kcpustat;
        u64 *cpustat = kcpustat.cpustat;
        u8 tail = __this_cpu_read(cpustat_tail);
        u16 sample_period_16 = get_16bit_precision(sample_period);

        kcpustat_cpu_fetch(&kcpustat, smp_processor_id());

        for (i = 0; i < NUM_STATS_PER_GROUP; i++) {
                old_stat = __this_cpu_read(cpustat_old[i]);
                new_stat = get_16bit_precision(cpustat[tracked_stats[i]]);
                util = DIV_ROUND_UP(100 * (new_stat - old_stat), sample_period_16);
                __this_cpu_write(cpustat_util[tail][i], util);
                __this_cpu_write(cpustat_old[i], new_stat);
        }

        __this_cpu_write(cpustat_tail, (tail + 1) % NUM_SAMPLE_PERIODS);
}

static void print_cpustat(void)
{
        int i, group;
        u8 tail = __this_cpu_read(cpustat_tail);
        u64 sample_period_second = sample_period;

        do_div(sample_period_second, NSEC_PER_SEC);

        /*
         * Outputting the "watchdog" prefix on every line is redundant and not
         * concise, and the original alarm information is sufficient for
         * positioning in logs, hence here printk() is used instead of pr_crit().
         */
        printk(KERN_CRIT "CPU#%d Utilization every %llus during lockup:\n",
               smp_processor_id(), sample_period_second);

        for (i = 0; i < NUM_SAMPLE_PERIODS; i++) {
                group = (tail + i) % NUM_SAMPLE_PERIODS;
                printk(KERN_CRIT "\t#%d: %3u%% system,\t%3u%% softirq,\t"
                        "%3u%% hardirq,\t%3u%% idle\n", i + 1,
                        __this_cpu_read(cpustat_util[group][STATS_SYSTEM]),
                        __this_cpu_read(cpustat_util[group][STATS_SOFTIRQ]),
                        __this_cpu_read(cpustat_util[group][STATS_HARDIRQ]),
                        __this_cpu_read(cpustat_util[group][STATS_IDLE]));
        }
}

#define HARDIRQ_PERCENT_THRESH          50
#define NUM_HARDIRQ_REPORT              5
struct irq_counts {
        int irq;
        u32 counts;
};

static DEFINE_PER_CPU(bool, snapshot_taken);

/* Tabulate the most frequent interrupts. */
static void tabulate_irq_count(struct irq_counts *irq_counts, int irq, u32 counts, int rank)
{
        int i;
        struct irq_counts new_count = {irq, counts};

        for (i = 0; i < rank; i++) {
                if (counts > irq_counts[i].counts)
                        swap(new_count, irq_counts[i]);
        }
}

/*
 * If the hardirq time exceeds HARDIRQ_PERCENT_THRESH% of the sample_period,
 * then the cause of softlockup might be interrupt storm. In this case, it
 * would be useful to start interrupt counting.
 */
static bool need_counting_irqs(void)
{
        u8 util;
        int tail = __this_cpu_read(cpustat_tail);

        tail = (tail + NUM_HARDIRQ_REPORT - 1) % NUM_HARDIRQ_REPORT;
        util = __this_cpu_read(cpustat_util[tail][STATS_HARDIRQ]);
        return util > HARDIRQ_PERCENT_THRESH;
}

static void start_counting_irqs(void)
{
        if (!__this_cpu_read(snapshot_taken)) {
                kstat_snapshot_irqs();
                __this_cpu_write(snapshot_taken, true);
        }
}

static void stop_counting_irqs(void)
{
        __this_cpu_write(snapshot_taken, false);
}

static void print_irq_counts(void)
{
        unsigned int i, count;
        struct irq_counts irq_counts_sorted[NUM_HARDIRQ_REPORT] = {
                {-1, 0}, {-1, 0}, {-1, 0}, {-1, 0}, {-1, 0}
        };

        if (__this_cpu_read(snapshot_taken)) {
                for_each_active_irq(i) {
                        count = kstat_get_irq_since_snapshot(i);
                        tabulate_irq_count(irq_counts_sorted, i, count, NUM_HARDIRQ_REPORT);
                }

                /*
                 * Outputting the "watchdog" prefix on every line is redundant and not
                 * concise, and the original alarm information is sufficient for
                 * positioning in logs, hence here printk() is used instead of pr_crit().
                 */
                printk(KERN_CRIT "CPU#%d Detect HardIRQ Time exceeds %d%%. Most frequent HardIRQs:\n",
                       smp_processor_id(), HARDIRQ_PERCENT_THRESH);

                for (i = 0; i < NUM_HARDIRQ_REPORT; i++) {
                        if (irq_counts_sorted[i].irq == -1)
                                break;

                        printk(KERN_CRIT "\t#%u: %-10u\tirq#%d\n",
                               i + 1, irq_counts_sorted[i].counts,
                               irq_counts_sorted[i].irq);
                }

                /*
                 * If the hardirq time is less than HARDIRQ_PERCENT_THRESH% in the last
                 * sample_period, then we suspect the interrupt storm might be subsiding.
                 */
                if (!need_counting_irqs())
                        stop_counting_irqs();
        }
}

static void report_cpu_status(void)
{
        print_cpustat();
        print_irq_counts();
}
#else
static inline void update_cpustat(void) { }
static inline void report_cpu_status(void) { }
static inline bool need_counting_irqs(void) { return false; }
static inline void start_counting_irqs(void) { }
static inline void stop_counting_irqs(void) { }
#endif

/*
 * Hard-lockup warnings should be triggered after just a few seconds. Soft-
 * lockups can have false positives under extreme conditions. So we generally
 * want a higher threshold for soft lockups than for hard lockups. So we couple
 * the thresholds with a factor: we make the soft threshold twice the amount of
 * time the hard threshold is.
 */
static int get_softlockup_thresh(void)
{
        return watchdog_thresh * 2;
}

/*
 * Returns seconds, approximately.  We don't need nanosecond
 * resolution, and we don't need to waste time with a big divide when
 * 2^30ns == 1.074s.
 */
static unsigned long get_timestamp(void)
{
        return running_clock() >> 30LL;  /* 2^30 ~= 10^9 */
}

static void set_sample_period(void)
{
        /*
         * convert watchdog_thresh from seconds to ns
         * the divide by 5 is to give hrtimer several chances (two
         * or three with the current relation between the soft
         * and hard thresholds) to increment before the
         * hardlockup detector generates a warning
         */
        sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / NUM_SAMPLE_PERIODS);
        watchdog_update_hrtimer_threshold(sample_period);
}

static void update_report_ts(void)
{
        __this_cpu_write(watchdog_report_ts, get_timestamp());
}

/* Commands for resetting the watchdog */
static void update_touch_ts(void)
{
        __this_cpu_write(watchdog_touch_ts, get_timestamp());
        update_report_ts();
}

/**
 * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
 *
 * Call when the scheduler may have stalled for legitimate reasons
 * preventing the watchdog task from executing - e.g. the scheduler
 * entering idle state.  This should only be used for scheduler events.
 * Use touch_softlockup_watchdog() for everything else.
 */
notrace void touch_softlockup_watchdog_sched(void)
{
        /*
         * Preemption can be enabled.  It doesn't matter which CPU's watchdog
         * report period gets restarted here, so use the raw_ operation.
         */
        raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}

notrace void touch_softlockup_watchdog(void)
{
        touch_softlockup_watchdog_sched();
        wq_watchdog_touch(raw_smp_processor_id());
}
EXPORT_SYMBOL(touch_softlockup_watchdog);

void touch_all_softlockup_watchdogs(void)
{
        int cpu;

        /*
         * watchdog_mutex cannpt be taken here, as this might be called
         * from (soft)interrupt context, so the access to
         * watchdog_allowed_cpumask might race with a concurrent update.
         *
         * The watchdog time stamp can race against a concurrent real
         * update as well, the only side effect might be a cycle delay for
         * the softlockup check.
         */
        for_each_cpu(cpu, &watchdog_allowed_mask) {
                per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT;
                wq_watchdog_touch(cpu);
        }
}

void touch_softlockup_watchdog_sync(void)
{
        __this_cpu_write(softlockup_touch_sync, true);
        __this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
}

static int is_softlockup(unsigned long touch_ts,
                         unsigned long period_ts,
                         unsigned long now)
{
        if ((watchdog_enabled & WATCHDOG_SOFTOCKUP_ENABLED) && watchdog_thresh) {
                /*
                 * If period_ts has not been updated during a sample_period, then
                 * in the subsequent few sample_periods, period_ts might also not
                 * be updated, which could indicate a potential softlockup. In
                 * this case, if we suspect the cause of the potential softlockup
                 * might be interrupt storm, then we need to count the interrupts
                 * to find which interrupt is storming.
                 */
                if (time_after_eq(now, period_ts + get_softlockup_thresh() / NUM_SAMPLE_PERIODS) &&
                    need_counting_irqs())
                        start_counting_irqs();

                /*
                 * A poorly behaving BPF scheduler can live-lock the system into
                 * soft lockups. Tell sched_ext to try ejecting the BPF
                 * scheduler when close to a soft lockup.
                 */
                if (time_after_eq(now, period_ts + get_softlockup_thresh() * 3 / 4))
                        scx_softlockup(now - touch_ts);

                /* Warn about unreasonable delays. */
                if (time_after(now, period_ts + get_softlockup_thresh()))
                        return now - touch_ts;
        }
        return 0;
}

/* watchdog detector functions */
static DEFINE_PER_CPU(struct completion, softlockup_completion);
static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work);

/*
 * The watchdog feed function - touches the timestamp.
 *
 * It only runs once every sample_period seconds (4 seconds by
 * default) to reset the softlockup timestamp. If this gets delayed
 * for more than 2*watchdog_thresh seconds then the debug-printout
 * triggers in watchdog_timer_fn().
 */
static int softlockup_fn(void *data)
{
        update_touch_ts();
        stop_counting_irqs();
        complete(this_cpu_ptr(&softlockup_completion));

        return 0;
}

/* watchdog kicker functions */
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
        unsigned long touch_ts, period_ts, now;
        struct pt_regs *regs = get_irq_regs();
        int duration;
        int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
        unsigned long flags;

        if (!watchdog_enabled)
                return HRTIMER_NORESTART;

        watchdog_hardlockup_kick();

        /* kick the softlockup detector */
        if (completion_done(this_cpu_ptr(&softlockup_completion))) {
                reinit_completion(this_cpu_ptr(&softlockup_completion));
                stop_one_cpu_nowait(smp_processor_id(),
                                softlockup_fn, NULL,
                                this_cpu_ptr(&softlockup_stop_work));
        }

        /* .. and repeat */
        hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));

        /*
         * Read the current timestamp first. It might become invalid anytime
         * when a virtual machine is stopped by the host or when the watchog
         * is touched from NMI.
         */
        now = get_timestamp();
        /*
         * If a virtual machine is stopped by the host it can look to
         * the watchdog like a soft lockup. This function touches the watchdog.
         */
        kvm_check_and_clear_guest_paused();
        /*
         * The stored timestamp is comparable with @now only when not touched.
         * It might get touched anytime from NMI. Make sure that is_softlockup()
         * uses the same (valid) value.
         */
        period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts));

        update_cpustat();

        /* Reset the interval when touched by known problematic code. */
        if (period_ts == SOFTLOCKUP_DELAY_REPORT) {
                if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
                        /*
                         * If the time stamp was touched atomically
                         * make sure the scheduler tick is up to date.
                         */
                        __this_cpu_write(softlockup_touch_sync, false);
                        sched_clock_tick();
                }

                update_report_ts();
                return HRTIMER_RESTART;
        }

        /* Check for a softlockup. */
        touch_ts = __this_cpu_read(watchdog_touch_ts);
        duration = is_softlockup(touch_ts, period_ts, now);
        if (unlikely(duration)) {
                /*
                 * Prevent multiple soft-lockup reports if one cpu is already
                 * engaged in dumping all cpu back traces.
                 */
                if (softlockup_all_cpu_backtrace) {
                        if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn))
                                return HRTIMER_RESTART;
                }

                /* Start period for the next softlockup warning. */
                update_report_ts();

                printk_cpu_sync_get_irqsave(flags);
                pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
                        smp_processor_id(), duration,
                        current->comm, task_pid_nr(current));
                report_cpu_status();
                print_modules();
                print_irqtrace_events(current);
                if (regs)
                        show_regs(regs);
                else
                        dump_stack();
                printk_cpu_sync_put_irqrestore(flags);

                if (softlockup_all_cpu_backtrace) {
                        trigger_allbutcpu_cpu_backtrace(smp_processor_id());
                        if (!softlockup_panic)
                                clear_bit_unlock(0, &soft_lockup_nmi_warn);
                }

                add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
                if (softlockup_panic)
                        panic("softlockup: hung tasks");
        }

        return HRTIMER_RESTART;
}

static void watchdog_enable(unsigned int cpu)
{
        struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
        struct completion *done = this_cpu_ptr(&softlockup_completion);

        WARN_ON_ONCE(cpu != smp_processor_id());

        init_completion(done);
        complete(done);

        /*
         * Start the timer first to prevent the hardlockup watchdog triggering
         * before the timer has a chance to fire.
         */
        hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
        hrtimer->function = watchdog_timer_fn;
        hrtimer_start(hrtimer, ns_to_ktime(sample_period),
                      HRTIMER_MODE_REL_PINNED_HARD);

        /* Initialize timestamp */
        update_touch_ts();
        /* Enable the hardlockup detector */
        if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED)
                watchdog_hardlockup_enable(cpu);
}

static void watchdog_disable(unsigned int cpu)
{
        struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);

        WARN_ON_ONCE(cpu != smp_processor_id());

        /*
         * Disable the hardlockup detector first. That prevents that a large
         * delay between disabling the timer and disabling the hardlockup
         * detector causes a false positive.
         */
        watchdog_hardlockup_disable(cpu);
        hrtimer_cancel(hrtimer);
        wait_for_completion(this_cpu_ptr(&softlockup_completion));
}

static int softlockup_stop_fn(void *data)
{
        watchdog_disable(smp_processor_id());
        return 0;
}

static void softlockup_stop_all(void)
{
        int cpu;

        if (!softlockup_initialized)
                return;

        for_each_cpu(cpu, &watchdog_allowed_mask)
                smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false);

        cpumask_clear(&watchdog_allowed_mask);
}

static int softlockup_start_fn(void *data)
{
        watchdog_enable(smp_processor_id());
        return 0;
}

static void softlockup_start_all(void)
{
        int cpu;

        cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask);
        for_each_cpu(cpu, &watchdog_allowed_mask)
                smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false);
}

int lockup_detector_online_cpu(unsigned int cpu)
{
        if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
                watchdog_enable(cpu);
        return 0;
}

int lockup_detector_offline_cpu(unsigned int cpu)
{
        if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
                watchdog_disable(cpu);
        return 0;
}

static void __lockup_detector_reconfigure(void)
{
        cpus_read_lock();
        watchdog_hardlockup_stop();

        softlockup_stop_all();
        set_sample_period();
        lockup_detector_update_enable();
        if (watchdog_enabled && watchdog_thresh)
                softlockup_start_all();

        watchdog_hardlockup_start();
        cpus_read_unlock();
        /*
         * Must be called outside the cpus locked section to prevent
         * recursive locking in the perf code.
         */
        __lockup_detector_cleanup();
}

void lockup_detector_reconfigure(void)
{
        mutex_lock(&watchdog_mutex);
        __lockup_detector_reconfigure();
        mutex_unlock(&watchdog_mutex);
}

/*
 * Create the watchdog infrastructure and configure the detector(s).
 */
static __init void lockup_detector_setup(void)
{
        /*
         * If sysctl is off and watchdog got disabled on the command line,
         * nothing to do here.
         */
        lockup_detector_update_enable();

        if (!IS_ENABLED(CONFIG_SYSCTL) &&
            !(watchdog_enabled && watchdog_thresh))
                return;

        mutex_lock(&watchdog_mutex);
        __lockup_detector_reconfigure();
        softlockup_initialized = true;
        mutex_unlock(&watchdog_mutex);
}

#else /* CONFIG_SOFTLOCKUP_DETECTOR */
static void __lockup_detector_reconfigure(void)
{
        cpus_read_lock();
        watchdog_hardlockup_stop();
        lockup_detector_update_enable();
        watchdog_hardlockup_start();
        cpus_read_unlock();
}
void lockup_detector_reconfigure(void)
{
        __lockup_detector_reconfigure();
}
static inline void lockup_detector_setup(void)
{
        __lockup_detector_reconfigure();
}
#endif /* !CONFIG_SOFTLOCKUP_DETECTOR */

static void __lockup_detector_cleanup(void)
{
        lockdep_assert_held(&watchdog_mutex);
        hardlockup_detector_perf_cleanup();
}

/**
 * lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes
 *
 * Caller must not hold the cpu hotplug rwsem.
 */
void lockup_detector_cleanup(void)
{
        mutex_lock(&watchdog_mutex);
        __lockup_detector_cleanup();
        mutex_unlock(&watchdog_mutex);
}

/**
 * lockup_detector_soft_poweroff - Interface to stop lockup detector(s)
 *
 * Special interface for parisc. It prevents lockup detector warnings from
 * the default pm_poweroff() function which busy loops forever.
 */
void lockup_detector_soft_poweroff(void)
{
        watchdog_enabled = 0;
}

#ifdef CONFIG_SYSCTL

/* Propagate any changes to the watchdog infrastructure */
static void proc_watchdog_update(void)
{
        /* Remove impossible cpus to keep sysctl output clean. */
        cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask);
        __lockup_detector_reconfigure();
}

/*
 * common function for watchdog, nmi_watchdog and soft_watchdog parameter
 *
 * caller             | table->data points to            | 'which'
 * -------------------|----------------------------------|-------------------------------
 * proc_watchdog      | watchdog_user_enabled            | WATCHDOG_HARDLOCKUP_ENABLED |
 *                    |                                  | WATCHDOG_SOFTOCKUP_ENABLED
 * -------------------|----------------------------------|-------------------------------
 * proc_nmi_watchdog  | watchdog_hardlockup_user_enabled | WATCHDOG_HARDLOCKUP_ENABLED
 * -------------------|----------------------------------|-------------------------------
 * proc_soft_watchdog | watchdog_softlockup_user_enabled | WATCHDOG_SOFTOCKUP_ENABLED
 */
static int proc_watchdog_common(int which, const struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        int err, old, *param = table->data;

        mutex_lock(&watchdog_mutex);

        old = *param;
        if (!write) {
                /*
                 * On read synchronize the userspace interface. This is a
                 * racy snapshot.
                 */
                *param = (watchdog_enabled & which) != 0;
                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
                *param = old;
        } else {
                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
                if (!err && old != READ_ONCE(*param))
                        proc_watchdog_update();
        }
        mutex_unlock(&watchdog_mutex);
        return err;
}

/*
 * /proc/sys/kernel/watchdog
 */
static int proc_watchdog(const struct ctl_table *table, int write,
                         void *buffer, size_t *lenp, loff_t *ppos)
{
        return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED |
                                    WATCHDOG_SOFTOCKUP_ENABLED,
                                    table, write, buffer, lenp, ppos);
}

/*
 * /proc/sys/kernel/nmi_watchdog
 */
static int proc_nmi_watchdog(const struct ctl_table *table, int write,
                             void *buffer, size_t *lenp, loff_t *ppos)
{
        if (!watchdog_hardlockup_available && write)
                return -ENOTSUPP;
        return proc_watchdog_common(WATCHDOG_HARDLOCKUP_ENABLED,
                                    table, write, buffer, lenp, ppos);
}

#ifdef CONFIG_SOFTLOCKUP_DETECTOR
/*
 * /proc/sys/kernel/soft_watchdog
 */
static int proc_soft_watchdog(const struct ctl_table *table, int write,
                              void *buffer, size_t *lenp, loff_t *ppos)
{
        return proc_watchdog_common(WATCHDOG_SOFTOCKUP_ENABLED,
                                    table, write, buffer, lenp, ppos);
}
#endif

/*
 * /proc/sys/kernel/watchdog_thresh
 */
static int proc_watchdog_thresh(const struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        int err, old;

        mutex_lock(&watchdog_mutex);

        old = READ_ONCE(watchdog_thresh);
        err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

        if (!err && write && old != READ_ONCE(watchdog_thresh))
                proc_watchdog_update();

        mutex_unlock(&watchdog_mutex);
        return err;
}

/*
 * The cpumask is the mask of possible cpus that the watchdog can run
 * on, not the mask of cpus it is actually running on.  This allows the
 * user to specify a mask that will include cpus that have not yet
 * been brought online, if desired.
 */
static int proc_watchdog_cpumask(const struct ctl_table *table, int write,
                                 void *buffer, size_t *lenp, loff_t *ppos)
{
        int err;

        mutex_lock(&watchdog_mutex);

        err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
        if (!err && write)
                proc_watchdog_update();

        mutex_unlock(&watchdog_mutex);
        return err;
}

static const int sixty = 60;

static struct ctl_table watchdog_sysctls[] = {
        {
                .procname       = "watchdog",
                .data                = &watchdog_user_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler   = proc_watchdog,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "watchdog_thresh",
                .data                = &watchdog_thresh,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_watchdog_thresh,
                .extra1                = SYSCTL_ZERO,
                .extra2                = (void *)&sixty,
        },
        {
                .procname        = "watchdog_cpumask",
                .data                = &watchdog_cpumask_bits,
                .maxlen                = NR_CPUS,
                .mode                = 0644,
                .proc_handler        = proc_watchdog_cpumask,
        },
#ifdef CONFIG_SOFTLOCKUP_DETECTOR
        {
                .procname       = "soft_watchdog",
                .data                = &watchdog_softlockup_user_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler   = proc_soft_watchdog,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "softlockup_panic",
                .data                = &softlockup_panic,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#ifdef CONFIG_SMP
        {
                .procname        = "softlockup_all_cpu_backtrace",
                .data                = &sysctl_softlockup_all_cpu_backtrace,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif /* CONFIG_SMP */
#endif
#ifdef CONFIG_HARDLOCKUP_DETECTOR
        {
                .procname        = "hardlockup_panic",
                .data                = &hardlockup_panic,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#ifdef CONFIG_SMP
        {
                .procname        = "hardlockup_all_cpu_backtrace",
                .data                = &sysctl_hardlockup_all_cpu_backtrace,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif /* CONFIG_SMP */
#endif
};

static struct ctl_table watchdog_hardlockup_sysctl[] = {
        {
                .procname       = "nmi_watchdog",
                .data                = &watchdog_hardlockup_user_enabled,
                .maxlen                = sizeof(int),
                .mode                = 0444,
                .proc_handler   = proc_nmi_watchdog,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
};

static void __init watchdog_sysctl_init(void)
{
        register_sysctl_init("kernel", watchdog_sysctls);

        if (watchdog_hardlockup_available)
                watchdog_hardlockup_sysctl[0].mode = 0644;
        register_sysctl_init("kernel", watchdog_hardlockup_sysctl);
}

#else
#define watchdog_sysctl_init() do { } while (0)
#endif /* CONFIG_SYSCTL */

static void __init lockup_detector_delay_init(struct work_struct *work);
static bool allow_lockup_detector_init_retry __initdata;

static struct work_struct detector_work __initdata =
                __WORK_INITIALIZER(detector_work, lockup_detector_delay_init);

static void __init lockup_detector_delay_init(struct work_struct *work)
{
        int ret;

        ret = watchdog_hardlockup_probe();
        if (ret) {
                if (ret == -ENODEV)
                        pr_info("NMI not fully supported\n");
                else
                        pr_info("Delayed init of the lockup detector failed: %d\n", ret);
                pr_info("Hard watchdog permanently disabled\n");
                return;
        }

        allow_lockup_detector_init_retry = false;

        watchdog_hardlockup_available = true;
        lockup_detector_setup();
}

/*
 * lockup_detector_retry_init - retry init lockup detector if possible.
 *
 * Retry hardlockup detector init. It is useful when it requires some
 * functionality that has to be initialized later on a particular
 * platform.
 */
void __init lockup_detector_retry_init(void)
{
        /* Must be called before late init calls */
        if (!allow_lockup_detector_init_retry)
                return;

        schedule_work(&detector_work);
}

/*
 * Ensure that optional delayed hardlockup init is proceed before
 * the init code and memory is freed.
 */
static int __init lockup_detector_check(void)
{
        /* Prevent any later retry. */
        allow_lockup_detector_init_retry = false;

        /* Make sure no work is pending. */
        flush_work(&detector_work);

        watchdog_sysctl_init();

        return 0;

}
late_initcall_sync(lockup_detector_check);

void __init lockup_detector_init(void)
{
        if (tick_nohz_full_enabled())
                pr_info("Disabling watchdog on nohz_full cores by default\n");

        cpumask_copy(&watchdog_cpumask,
                     housekeeping_cpumask(HK_TYPE_TIMER));

        if (!watchdog_hardlockup_probe())
                watchdog_hardlockup_available = true;
        else
                allow_lockup_detector_init_retry = true;

        lockup_detector_setup();
}






















































    1 

    1 



    3 








    1 
    1 

    1 







    2 
    2 













    1 










    1 











    1 











    1 











    1 













    1 













    1 













    1 











    1 












    1 
























    1 
    1 



















































































    4 
















    4 









   10 







    3 







    6 







    2 







    4 






    3 







    8 








    7 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __CFG802154_RDEV_OPS
#define __CFG802154_RDEV_OPS

#include <net/cfg802154.h>

#include "core.h"
#include "trace.h"

static inline struct net_device *
rdev_add_virtual_intf_deprecated(struct cfg802154_registered_device *rdev,
                                 const char *name,
                                 unsigned char name_assign_type,
                                 int type)
{
        return rdev->ops->add_virtual_intf_deprecated(&rdev->wpan_phy, name,
                                                      name_assign_type, type);
}

static inline void
rdev_del_virtual_intf_deprecated(struct cfg802154_registered_device *rdev,
                                 struct net_device *dev)
{
        rdev->ops->del_virtual_intf_deprecated(&rdev->wpan_phy, dev);
}

static inline int
rdev_suspend(struct cfg802154_registered_device *rdev)
{
        int ret;
        trace_802154_rdev_suspend(&rdev->wpan_phy);
        ret = rdev->ops->suspend(&rdev->wpan_phy);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_resume(struct cfg802154_registered_device *rdev)
{
        int ret;
        trace_802154_rdev_resume(&rdev->wpan_phy);
        ret = rdev->ops->resume(&rdev->wpan_phy);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_add_virtual_intf(struct cfg802154_registered_device *rdev, char *name,
                      unsigned char name_assign_type,
                      enum nl802154_iftype type, __le64 extended_addr)
{
        int ret;

        trace_802154_rdev_add_virtual_intf(&rdev->wpan_phy, name, type,
                                           extended_addr);
        ret = rdev->ops->add_virtual_intf(&rdev->wpan_phy, name,
                                          name_assign_type, type,
                                          extended_addr);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_del_virtual_intf(struct cfg802154_registered_device *rdev,
                      struct wpan_dev *wpan_dev)
{
        int ret;

        trace_802154_rdev_del_virtual_intf(&rdev->wpan_phy, wpan_dev);
        ret = rdev->ops->del_virtual_intf(&rdev->wpan_phy, wpan_dev);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_channel(struct cfg802154_registered_device *rdev, u8 page, u8 channel)
{
        int ret;

        trace_802154_rdev_set_channel(&rdev->wpan_phy, page, channel);
        ret = rdev->ops->set_channel(&rdev->wpan_phy, page, channel);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_cca_mode(struct cfg802154_registered_device *rdev,
                  const struct wpan_phy_cca *cca)
{
        int ret;

        trace_802154_rdev_set_cca_mode(&rdev->wpan_phy, cca);
        ret = rdev->ops->set_cca_mode(&rdev->wpan_phy, cca);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_cca_ed_level(struct cfg802154_registered_device *rdev, s32 ed_level)
{
        int ret;

        trace_802154_rdev_set_cca_ed_level(&rdev->wpan_phy, ed_level);
        ret = rdev->ops->set_cca_ed_level(&rdev->wpan_phy, ed_level);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_tx_power(struct cfg802154_registered_device *rdev,
                  s32 power)
{
        int ret;

        trace_802154_rdev_set_tx_power(&rdev->wpan_phy, power);
        ret = rdev->ops->set_tx_power(&rdev->wpan_phy, power);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_pan_id(struct cfg802154_registered_device *rdev,
                struct wpan_dev *wpan_dev, __le16 pan_id)
{
        int ret;

        trace_802154_rdev_set_pan_id(&rdev->wpan_phy, wpan_dev, pan_id);
        ret = rdev->ops->set_pan_id(&rdev->wpan_phy, wpan_dev, pan_id);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_short_addr(struct cfg802154_registered_device *rdev,
                    struct wpan_dev *wpan_dev, __le16 short_addr)
{
        int ret;

        trace_802154_rdev_set_short_addr(&rdev->wpan_phy, wpan_dev, short_addr);
        ret = rdev->ops->set_short_addr(&rdev->wpan_phy, wpan_dev, short_addr);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_backoff_exponent(struct cfg802154_registered_device *rdev,
                          struct wpan_dev *wpan_dev, u8 min_be, u8 max_be)
{
        int ret;

        trace_802154_rdev_set_backoff_exponent(&rdev->wpan_phy, wpan_dev,
                                               min_be, max_be);
        ret = rdev->ops->set_backoff_exponent(&rdev->wpan_phy, wpan_dev,
                                              min_be, max_be);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_max_csma_backoffs(struct cfg802154_registered_device *rdev,
                           struct wpan_dev *wpan_dev, u8 max_csma_backoffs)
{
        int ret;

        trace_802154_rdev_set_csma_backoffs(&rdev->wpan_phy, wpan_dev,
                                            max_csma_backoffs);
        ret = rdev->ops->set_max_csma_backoffs(&rdev->wpan_phy, wpan_dev,
                                               max_csma_backoffs);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_max_frame_retries(struct cfg802154_registered_device *rdev,
                           struct wpan_dev *wpan_dev, s8 max_frame_retries)
{
        int ret;

        trace_802154_rdev_set_max_frame_retries(&rdev->wpan_phy, wpan_dev,
                                                max_frame_retries);
        ret = rdev->ops->set_max_frame_retries(&rdev->wpan_phy, wpan_dev,
                                               max_frame_retries);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_lbt_mode(struct cfg802154_registered_device *rdev,
                  struct wpan_dev *wpan_dev, bool mode)
{
        int ret;

        trace_802154_rdev_set_lbt_mode(&rdev->wpan_phy, wpan_dev, mode);
        ret = rdev->ops->set_lbt_mode(&rdev->wpan_phy, wpan_dev, mode);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int
rdev_set_ackreq_default(struct cfg802154_registered_device *rdev,
                        struct wpan_dev *wpan_dev, bool ackreq)
{
        int ret;

        trace_802154_rdev_set_ackreq_default(&rdev->wpan_phy, wpan_dev,
                                             ackreq);
        ret = rdev->ops->set_ackreq_default(&rdev->wpan_phy, wpan_dev, ackreq);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int rdev_trigger_scan(struct cfg802154_registered_device *rdev,
                                    struct cfg802154_scan_request *request)
{
        int ret;

        if (!rdev->ops->trigger_scan)
                return -EOPNOTSUPP;

        trace_802154_rdev_trigger_scan(&rdev->wpan_phy, request);
        ret = rdev->ops->trigger_scan(&rdev->wpan_phy, request);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int rdev_abort_scan(struct cfg802154_registered_device *rdev,
                                  struct wpan_dev *wpan_dev)
{
        int ret;

        if (!rdev->ops->abort_scan)
                return -EOPNOTSUPP;

        trace_802154_rdev_abort_scan(&rdev->wpan_phy, wpan_dev);
        ret = rdev->ops->abort_scan(&rdev->wpan_phy, wpan_dev);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int rdev_send_beacons(struct cfg802154_registered_device *rdev,
                                    struct cfg802154_beacon_request *request)
{
        int ret;

        if (!rdev->ops->send_beacons)
                return -EOPNOTSUPP;

        trace_802154_rdev_send_beacons(&rdev->wpan_phy, request);
        ret = rdev->ops->send_beacons(&rdev->wpan_phy, request);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int rdev_stop_beacons(struct cfg802154_registered_device *rdev,
                                    struct wpan_dev *wpan_dev)
{
        int ret;

        if (!rdev->ops->stop_beacons)
                return -EOPNOTSUPP;

        trace_802154_rdev_stop_beacons(&rdev->wpan_phy, wpan_dev);
        ret = rdev->ops->stop_beacons(&rdev->wpan_phy, wpan_dev);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int rdev_associate(struct cfg802154_registered_device *rdev,
                                 struct wpan_dev *wpan_dev,
                                 struct ieee802154_addr *coord)
{
        int ret;

        if (!rdev->ops->associate)
                return -EOPNOTSUPP;

        trace_802154_rdev_associate(&rdev->wpan_phy, wpan_dev, coord);
        ret = rdev->ops->associate(&rdev->wpan_phy, wpan_dev, coord);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

static inline int rdev_disassociate(struct cfg802154_registered_device *rdev,
                                    struct wpan_dev *wpan_dev,
                                    struct ieee802154_addr *target)
{
        int ret;

        if (!rdev->ops->disassociate)
                return -EOPNOTSUPP;

        trace_802154_rdev_disassociate(&rdev->wpan_phy, wpan_dev, target);
        ret = rdev->ops->disassociate(&rdev->wpan_phy, wpan_dev, target);
        trace_802154_rdev_return_int(&rdev->wpan_phy, ret);
        return ret;
}

#ifdef CONFIG_IEEE802154_NL802154_EXPERIMENTAL
/* TODO this is already a nl802154, so move into ieee802154 */
static inline void
rdev_get_llsec_table(struct cfg802154_registered_device *rdev,
                     struct wpan_dev *wpan_dev,
                     struct ieee802154_llsec_table **table)
{
        rdev->ops->get_llsec_table(&rdev->wpan_phy, wpan_dev, table);
}

static inline void
rdev_lock_llsec_table(struct cfg802154_registered_device *rdev,
                      struct wpan_dev *wpan_dev)
{
        rdev->ops->lock_llsec_table(&rdev->wpan_phy, wpan_dev);
}

static inline void
rdev_unlock_llsec_table(struct cfg802154_registered_device *rdev,
                        struct wpan_dev *wpan_dev)
{
        rdev->ops->unlock_llsec_table(&rdev->wpan_phy, wpan_dev);
}

static inline int
rdev_get_llsec_params(struct cfg802154_registered_device *rdev,
                      struct wpan_dev *wpan_dev,
                      struct ieee802154_llsec_params *params)
{
        return rdev->ops->get_llsec_params(&rdev->wpan_phy, wpan_dev, params);
}

static inline int
rdev_set_llsec_params(struct cfg802154_registered_device *rdev,
                      struct wpan_dev *wpan_dev,
                      const struct ieee802154_llsec_params *params,
                      u32 changed)
{
        return rdev->ops->set_llsec_params(&rdev->wpan_phy, wpan_dev, params,
                                           changed);
}

static inline int
rdev_add_llsec_key(struct cfg802154_registered_device *rdev,
                   struct wpan_dev *wpan_dev,
                   const struct ieee802154_llsec_key_id *id,
                   const struct ieee802154_llsec_key *key)
{
        return rdev->ops->add_llsec_key(&rdev->wpan_phy, wpan_dev, id, key);
}

static inline int
rdev_del_llsec_key(struct cfg802154_registered_device *rdev,
                   struct wpan_dev *wpan_dev,
                   const struct ieee802154_llsec_key_id *id)
{
        return rdev->ops->del_llsec_key(&rdev->wpan_phy, wpan_dev, id);
}

static inline int
rdev_add_seclevel(struct cfg802154_registered_device *rdev,
                  struct wpan_dev *wpan_dev,
                  const struct ieee802154_llsec_seclevel *sl)
{
        return rdev->ops->add_seclevel(&rdev->wpan_phy, wpan_dev, sl);
}

static inline int
rdev_del_seclevel(struct cfg802154_registered_device *rdev,
                  struct wpan_dev *wpan_dev,
                  const struct ieee802154_llsec_seclevel *sl)
{
        return rdev->ops->del_seclevel(&rdev->wpan_phy, wpan_dev, sl);
}

static inline int
rdev_add_device(struct cfg802154_registered_device *rdev,
                struct wpan_dev *wpan_dev,
                const struct ieee802154_llsec_device *dev_desc)
{
        return rdev->ops->add_device(&rdev->wpan_phy, wpan_dev, dev_desc);
}

static inline int
rdev_del_device(struct cfg802154_registered_device *rdev,
                struct wpan_dev *wpan_dev, __le64 extended_addr)
{
        return rdev->ops->del_device(&rdev->wpan_phy, wpan_dev, extended_addr);
}

static inline int
rdev_add_devkey(struct cfg802154_registered_device *rdev,
                struct wpan_dev *wpan_dev, __le64 extended_addr,
                const struct ieee802154_llsec_device_key *devkey)
{
        return rdev->ops->add_devkey(&rdev->wpan_phy, wpan_dev, extended_addr,
                                     devkey);
}

static inline int
rdev_del_devkey(struct cfg802154_registered_device *rdev,
                struct wpan_dev *wpan_dev, __le64 extended_addr,
                const struct ieee802154_llsec_device_key *devkey)
{
        return rdev->ops->del_devkey(&rdev->wpan_phy, wpan_dev, extended_addr,
                                     devkey);
}
#endif /* CONFIG_IEEE802154_NL802154_EXPERIMENTAL */

#endif /* __CFG802154_RDEV_OPS */













































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 




    2 
    1 
    1 





    2 















































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
// SPDX-License-Identifier: GPL-2.0+
/*
 * Edgeport USB Serial Converter driver
 *
 * Copyright (C) 2000-2002 Inside Out Networks, All rights reserved.
 * Copyright (C) 2001-2002 Greg Kroah-Hartman <greg@kroah.com>
 *
 * Supports the following devices:
 *        EP/1 EP/2 EP/4 EP/21 EP/22 EP/221 EP/42 EP/421 WATCHPORT
 *
 * For questions or problems with this driver, contact Inside Out
 * Networks technical support, or Peter Berger <pberger@brimson.com>,
 * or Al Borchers <alborchers@steinerpoint.com>.
 */

#include <linux/kernel.h>
#include <linux/jiffies.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/tty.h>
#include <linux/tty_driver.h>
#include <linux/tty_flip.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/serial.h>
#include <linux/swab.h>
#include <linux/kfifo.h>
#include <linux/ioctl.h>
#include <linux/firmware.h>
#include <linux/uaccess.h>
#include <linux/usb.h>
#include <linux/usb/serial.h>

#include "io_16654.h"
#include "io_usbvend.h"
#include "io_ti.h"

#define DRIVER_AUTHOR "Greg Kroah-Hartman <greg@kroah.com> and David Iacovelli"
#define DRIVER_DESC "Edgeport USB Serial Driver"

#define EPROM_PAGE_SIZE                64


/* different hardware types */
#define HARDWARE_TYPE_930        0
#define HARDWARE_TYPE_TIUMP        1

/* IOCTL_PRIVATE_TI_GET_MODE Definitions */
#define        TI_MODE_CONFIGURING        0   /* Device has not entered start device */
#define        TI_MODE_BOOT                1   /* Staying in boot mode                   */
#define TI_MODE_DOWNLOAD        2   /* Made it to download mode                   */
#define TI_MODE_TRANSITIONING        3   /*
                                     * Currently in boot mode but
                                     * transitioning to download mode
                                     */

/* read urb state */
#define EDGE_READ_URB_RUNNING        0
#define EDGE_READ_URB_STOPPING        1
#define EDGE_READ_URB_STOPPED        2


/* Product information read from the Edgeport */
struct product_info {
        int        TiMode;                        /* Current TI Mode  */
        u8        hardware_type;                /* Type of hardware */
} __packed;

/*
 * Edgeport firmware header
 *
 * "build_number" has been set to 0 in all three of the images I have
 * seen, and Digi Tech Support suggests that it is safe to ignore it.
 *
 * "length" is the number of bytes of actual data following the header.
 *
 * "checksum" is the low order byte resulting from adding the values of
 * all the data bytes.
 */
struct edgeport_fw_hdr {
        u8 major_version;
        u8 minor_version;
        __le16 build_number;
        __le16 length;
        u8 checksum;
} __packed;

struct edgeport_port {
        u16 uart_base;
        u16 dma_address;
        u8 shadow_msr;
        u8 shadow_mcr;
        u8 shadow_lsr;
        u8 lsr_mask;
        u32 ump_read_timeout;                /*
                                         * Number of milliseconds the UMP will
                                         * wait without data before completing
                                         * a read short
                                         */
        int baud_rate;
        int close_pending;
        int lsr_event;

        struct edgeport_serial        *edge_serial;
        struct usb_serial_port        *port;
        u8 bUartMode;                /* Port type, 0: RS232, etc. */
        spinlock_t ep_lock;
        int ep_read_urb_state;
        int ep_write_urb_in_use;
};

struct edgeport_serial {
        struct product_info product_info;
        u8 TI_I2C_Type;                        /* Type of I2C in UMP */
        u8 TiReadI2C;                        /*
                                         * Set to TRUE if we have read the
                                         * I2c in Boot Mode
                                         */
        struct mutex es_lock;
        int num_ports_open;
        struct usb_serial *serial;
        struct delayed_work heartbeat_work;
        int fw_version;
        bool use_heartbeat;
};


/* Devices that this driver supports */
static const struct usb_device_id edgeport_1port_id_table[] = {
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_1) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_TI3410_EDGEPORT_1) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_TI3410_EDGEPORT_1I) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_PROXIMITY) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_MOTION) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_MOISTURE) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_TEMPERATURE) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_HUMIDITY) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_POWER) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_LIGHT) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_RADIATION) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_DISTANCE) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_ACCELERATION) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_PROX_DIST) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_PLUS_PWR_HP4CD) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_PLUS_PWR_PCI) },
        { }
};

static const struct usb_device_id edgeport_2port_id_table[] = {
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_2) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_2C) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_2I) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_421) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_21) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_42) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_4) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_4I) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_22I) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_221C) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_22C) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_21C) },
        /* The 4, 8 and 16 port devices show up as multiple 2 port devices */
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_4S) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_8) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_8S) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_416) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_416B) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_E5805A) },
        { }
};

/* Devices that this driver supports */
static const struct usb_device_id id_table_combined[] = {
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_1) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_TI3410_EDGEPORT_1) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_TI3410_EDGEPORT_1I) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_PROXIMITY) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_MOTION) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_MOISTURE) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_TEMPERATURE) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_HUMIDITY) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_POWER) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_LIGHT) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_RADIATION) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_DISTANCE) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_ACCELERATION) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_WP_PROX_DIST) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_PLUS_PWR_HP4CD) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_PLUS_PWR_PCI) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_2) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_2C) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_2I) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_421) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_21) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_42) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_4) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_4I) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_22I) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_221C) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_22C) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_21C) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_4S) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_8) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_8S) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_416) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_416B) },
        { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_E5805A) },
        { }
};

MODULE_DEVICE_TABLE(usb, id_table_combined);

static bool ignore_cpu_rev;
static int default_uart_mode;                /* RS232 */

static void edge_tty_recv(struct usb_serial_port *port, unsigned char *data,
                int length);

static void stop_read(struct edgeport_port *edge_port);
static int restart_read(struct edgeport_port *edge_port);

static void edge_set_termios(struct tty_struct *tty,
                             struct usb_serial_port *port,
                             const struct ktermios *old_termios);
static void edge_send(struct usb_serial_port *port, struct tty_struct *tty);

static int do_download_mode(struct edgeport_serial *serial,
                const struct firmware *fw);
static int do_boot_mode(struct edgeport_serial *serial,
                const struct firmware *fw);

/* sysfs attributes */
static int edge_create_sysfs_attrs(struct usb_serial_port *port);
static int edge_remove_sysfs_attrs(struct usb_serial_port *port);

/*
 * Some release of Edgeport firmware "down3.bin" after version 4.80
 * introduced code to automatically disconnect idle devices on some
 * Edgeport models after periods of inactivity, typically ~60 seconds.
 * This occurs without regard to whether ports on the device are open
 * or not.  Digi International Tech Support suggested:
 *
 * 1.  Adding driver "heartbeat" code to reset the firmware timer by
 *     requesting a descriptor record every 15 seconds, which should be
 *     effective with newer firmware versions that require it, and benign
 *     with older versions that do not. In practice 40 seconds seems often
 *     enough.
 * 2.  The heartbeat code is currently required only on Edgeport/416 models.
 */
#define FW_HEARTBEAT_VERSION_CUTOFF ((4 << 8) + 80)
#define FW_HEARTBEAT_SECS 40

/* Timeouts in msecs: firmware downloads take longer */
#define TI_VSEND_TIMEOUT_DEFAULT 1000
#define TI_VSEND_TIMEOUT_FW_DOWNLOAD 10000

static int ti_vread_sync(struct usb_device *dev, u8 request, u16 value,
                u16 index, void *data, int size)
{
        int status;

        status = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0), request,
                        (USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN),
                        value, index, data, size, 1000);
        if (status < 0)
                return status;
        if (status != size) {
                dev_dbg(&dev->dev, "%s - wanted to read %d, but only read %d\n",
                        __func__, size, status);
                return -ECOMM;
        }
        return 0;
}

static int ti_vsend_sync(struct usb_device *dev, u8 request, u16 value,
                u16 index, void *data, int size, int timeout)
{
        int status;

        status = usb_control_msg(dev, usb_sndctrlpipe(dev, 0), request,
                        (USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT),
                        value, index, data, size, timeout);
        if (status < 0)
                return status;

        return 0;
}

static int read_port_cmd(struct usb_serial_port *port, u8 command, u16 value,
                void *data, int size)
{
        return ti_vread_sync(port->serial->dev, command, value,
                        UMPM_UART1_PORT + port->port_number,
                        data, size);
}

static int send_port_cmd(struct usb_serial_port *port, u8 command, u16 value,
                void *data, int size)
{
        return ti_vsend_sync(port->serial->dev, command, value,
                        UMPM_UART1_PORT + port->port_number,
                        data, size, TI_VSEND_TIMEOUT_DEFAULT);
}

/* clear tx/rx buffers and fifo in TI UMP */
static int purge_port(struct usb_serial_port *port, u16 mask)
{
        int port_number = port->port_number;

        dev_dbg(&port->dev, "%s - port %d, mask %x\n", __func__, port_number, mask);

        return send_port_cmd(port, UMPC_PURGE_PORT, mask, NULL, 0);
}

/**
 * read_download_mem - Read edgeport memory from TI chip
 * @dev: usb device pointer
 * @start_address: Device CPU address at which to read
 * @length: Length of above data
 * @address_type: Can read both XDATA and I2C
 * @buffer: pointer to input data buffer
 */
static int read_download_mem(struct usb_device *dev, int start_address,
                                int length, u8 address_type, u8 *buffer)
{
        int status = 0;
        u8 read_length;
        u16 be_start_address;

        dev_dbg(&dev->dev, "%s - @ %x for %d\n", __func__, start_address, length);

        /*
         * Read in blocks of 64 bytes
         * (TI firmware can't handle more than 64 byte reads)
         */
        while (length) {
                if (length > 64)
                        read_length = 64;
                else
                        read_length = (u8)length;

                if (read_length > 1) {
                        dev_dbg(&dev->dev, "%s - @ %x for %d\n", __func__, start_address, read_length);
                }
                /*
                 * NOTE: Must use swab as wIndex is sent in little-endian
                 *       byte order regardless of host byte order.
                 */
                be_start_address = swab16((u16)start_address);
                status = ti_vread_sync(dev, UMPC_MEMORY_READ,
                                        (u16)address_type,
                                        be_start_address,
                                        buffer, read_length);

                if (status) {
                        dev_dbg(&dev->dev, "%s - ERROR %x\n", __func__, status);
                        return status;
                }

                if (read_length > 1)
                        usb_serial_debug_data(&dev->dev, __func__, read_length, buffer);

                /* Update pointers/length */
                start_address += read_length;
                buffer += read_length;
                length -= read_length;
        }

        return status;
}

static int read_ram(struct usb_device *dev, int start_address,
                                                int length, u8 *buffer)
{
        return read_download_mem(dev, start_address, length,
                                        DTK_ADDR_SPACE_XDATA, buffer);
}

/* Read edgeport memory to a given block */
static int read_boot_mem(struct edgeport_serial *serial,
                                int start_address, int length, u8 *buffer)
{
        int status = 0;
        int i;

        for (i = 0; i < length; i++) {
                status = ti_vread_sync(serial->serial->dev,
                                UMPC_MEMORY_READ, serial->TI_I2C_Type,
                                (u16)(start_address+i), &buffer[i], 0x01);
                if (status) {
                        dev_dbg(&serial->serial->dev->dev, "%s - ERROR %x\n", __func__, status);
                        return status;
                }
        }

        dev_dbg(&serial->serial->dev->dev, "%s - start_address = %x, length = %d\n",
                __func__, start_address, length);
        usb_serial_debug_data(&serial->serial->dev->dev, __func__, length, buffer);

        serial->TiReadI2C = 1;

        return status;
}

/* Write given block to TI EPROM memory */
static int write_boot_mem(struct edgeport_serial *serial,
                                int start_address, int length, u8 *buffer)
{
        int status = 0;
        int i;
        u8 *temp;

        /* Must do a read before write */
        if (!serial->TiReadI2C) {
                temp = kmalloc(1, GFP_KERNEL);
                if (!temp)
                        return -ENOMEM;

                status = read_boot_mem(serial, 0, 1, temp);
                kfree(temp);
                if (status)
                        return status;
        }

        for (i = 0; i < length; ++i) {
                status = ti_vsend_sync(serial->serial->dev, UMPC_MEMORY_WRITE,
                                buffer[i], (u16)(i + start_address), NULL,
                                0, TI_VSEND_TIMEOUT_DEFAULT);
                if (status)
                        return status;
        }

        dev_dbg(&serial->serial->dev->dev, "%s - start_sddr = %x, length = %d\n", __func__, start_address, length);
        usb_serial_debug_data(&serial->serial->dev->dev, __func__, length, buffer);

        return status;
}

/* Write edgeport I2C memory to TI chip        */
static int write_i2c_mem(struct edgeport_serial *serial,
                int start_address, int length, u8 address_type, u8 *buffer)
{
        struct device *dev = &serial->serial->dev->dev;
        int status = 0;
        int write_length;
        u16 be_start_address;

        /* We can only send a maximum of 1 aligned byte page at a time */

        /* calculate the number of bytes left in the first page */
        write_length = EPROM_PAGE_SIZE -
                                (start_address & (EPROM_PAGE_SIZE - 1));

        if (write_length > length)
                write_length = length;

        dev_dbg(dev, "%s - BytesInFirstPage Addr = %x, length = %d\n",
                __func__, start_address, write_length);
        usb_serial_debug_data(dev, __func__, write_length, buffer);

        /*
         * Write first page.
         *
         * NOTE: Must use swab as wIndex is sent in little-endian byte order
         *       regardless of host byte order.
         */
        be_start_address = swab16((u16)start_address);
        status = ti_vsend_sync(serial->serial->dev, UMPC_MEMORY_WRITE,
                                (u16)address_type, be_start_address,
                                buffer,        write_length, TI_VSEND_TIMEOUT_DEFAULT);
        if (status) {
                dev_dbg(dev, "%s - ERROR %d\n", __func__, status);
                return status;
        }

        length                -= write_length;
        start_address        += write_length;
        buffer                += write_length;

        /*
         * We should be aligned now -- can write max page size bytes at a
         * time.
         */
        while (length) {
                if (length > EPROM_PAGE_SIZE)
                        write_length = EPROM_PAGE_SIZE;
                else
                        write_length = length;

                dev_dbg(dev, "%s - Page Write Addr = %x, length = %d\n",
                        __func__, start_address, write_length);
                usb_serial_debug_data(dev, __func__, write_length, buffer);

                /*
                 * Write next page.
                 *
                 * NOTE: Must use swab as wIndex is sent in little-endian byte
                 *       order regardless of host byte order.
                 */
                be_start_address = swab16((u16)start_address);
                status = ti_vsend_sync(serial->serial->dev, UMPC_MEMORY_WRITE,
                                (u16)address_type, be_start_address, buffer,
                                write_length, TI_VSEND_TIMEOUT_DEFAULT);
                if (status) {
                        dev_err(dev, "%s - ERROR %d\n", __func__, status);
                        return status;
                }

                length                -= write_length;
                start_address        += write_length;
                buffer                += write_length;
        }
        return status;
}

/*
 * Examine the UMP DMA registers and LSR
 *
 * Check the MSBit of the X and Y DMA byte count registers.
 * A zero in this bit indicates that the TX DMA buffers are empty
 * then check the TX Empty bit in the UART.
 */
static int tx_active(struct edgeport_port *port)
{
        int status;
        struct out_endpoint_desc_block *oedb;
        u8 *lsr;
        int bytes_left = 0;

        oedb = kmalloc(sizeof(*oedb), GFP_KERNEL);
        if (!oedb)
                return -ENOMEM;

        /*
         * Sigh, that's right, just one byte, as not all platforms can
         * do DMA from stack
         */
        lsr = kmalloc(1, GFP_KERNEL);
        if (!lsr) {
                kfree(oedb);
                return -ENOMEM;
        }
        /* Read the DMA Count Registers */
        status = read_ram(port->port->serial->dev, port->dma_address,
                                                sizeof(*oedb), (void *)oedb);
        if (status)
                goto exit_is_tx_active;

        dev_dbg(&port->port->dev, "%s - XByteCount    0x%X\n", __func__, oedb->XByteCount);

        /* and the LSR */
        status = read_ram(port->port->serial->dev,
                        port->uart_base + UMPMEM_OFFS_UART_LSR, 1, lsr);

        if (status)
                goto exit_is_tx_active;
        dev_dbg(&port->port->dev, "%s - LSR = 0x%X\n", __func__, *lsr);

        /* If either buffer has data or we are transmitting then return TRUE */
        if ((oedb->XByteCount & 0x80) != 0)
                bytes_left += 64;

        if ((*lsr & UMP_UART_LSR_TX_MASK) == 0)
                bytes_left += 1;

        /* We return Not Active if we get any kind of error */
exit_is_tx_active:
        dev_dbg(&port->port->dev, "%s - return %d\n", __func__, bytes_left);

        kfree(lsr);
        kfree(oedb);
        return bytes_left;
}

static int choose_config(struct usb_device *dev)
{
        /*
         * There may be multiple configurations on this device, in which case
         * we would need to read and parse all of them to find out which one
         * we want. However, we just support one config at this point,
         * configuration # 1, which is Config Descriptor 0.
         */

        dev_dbg(&dev->dev, "%s - Number of Interfaces = %d\n",
                __func__, dev->config->desc.bNumInterfaces);
        dev_dbg(&dev->dev, "%s - MAX Power            = %d\n",
                __func__, dev->config->desc.bMaxPower * 2);

        if (dev->config->desc.bNumInterfaces != 1) {
                dev_err(&dev->dev, "%s - bNumInterfaces is not 1, ERROR!\n", __func__);
                return -ENODEV;
        }

        return 0;
}

static int read_rom(struct edgeport_serial *serial,
                                int start_address, int length, u8 *buffer)
{
        int status;

        if (serial->product_info.TiMode == TI_MODE_DOWNLOAD) {
                status = read_download_mem(serial->serial->dev,
                                               start_address,
                                               length,
                                               serial->TI_I2C_Type,
                                               buffer);
        } else {
                status = read_boot_mem(serial, start_address, length,
                                                                buffer);
        }
        return status;
}

static int write_rom(struct edgeport_serial *serial, int start_address,
                                                int length, u8 *buffer)
{
        if (serial->product_info.TiMode == TI_MODE_BOOT)
                return write_boot_mem(serial, start_address, length,
                                                                buffer);

        if (serial->product_info.TiMode == TI_MODE_DOWNLOAD)
                return write_i2c_mem(serial, start_address, length,
                                                serial->TI_I2C_Type, buffer);
        return -EINVAL;
}

/* Read a descriptor header from I2C based on type */
static int get_descriptor_addr(struct edgeport_serial *serial,
                                int desc_type, struct ti_i2c_desc *rom_desc)
{
        int start_address;
        int status;

        /* Search for requested descriptor in I2C */
        start_address = 2;
        do {
                status = read_rom(serial,
                                   start_address,
                                   sizeof(struct ti_i2c_desc),
                                   (u8 *)rom_desc);
                if (status)
                        return 0;

                if (rom_desc->Type == desc_type)
                        return start_address;

                start_address = start_address + sizeof(struct ti_i2c_desc) +
                                                le16_to_cpu(rom_desc->Size);

        } while ((start_address < TI_MAX_I2C_SIZE) && rom_desc->Type);

        return 0;
}

/* Validate descriptor checksum */
static int valid_csum(struct ti_i2c_desc *rom_desc, u8 *buffer)
{
        u16 i;
        u8 cs = 0;

        for (i = 0; i < le16_to_cpu(rom_desc->Size); i++)
                cs = (u8)(cs + buffer[i]);

        if (cs != rom_desc->CheckSum) {
                pr_debug("%s - Mismatch %x - %x", __func__, rom_desc->CheckSum, cs);
                return -EINVAL;
        }
        return 0;
}

/* Make sure that the I2C image is good */
static int check_i2c_image(struct edgeport_serial *serial)
{
        struct device *dev = &serial->serial->dev->dev;
        int status = 0;
        struct ti_i2c_desc *rom_desc;
        int start_address = 2;
        u8 *buffer;
        u16 ttype;

        rom_desc = kmalloc(sizeof(*rom_desc), GFP_KERNEL);
        if (!rom_desc)
                return -ENOMEM;

        buffer = kmalloc(TI_MAX_I2C_SIZE, GFP_KERNEL);
        if (!buffer) {
                kfree(rom_desc);
                return -ENOMEM;
        }

        /* Read the first byte (Signature0) must be 0x52 or 0x10 */
        status = read_rom(serial, 0, 1, buffer);
        if (status)
                goto out;

        if (*buffer != UMP5152 && *buffer != UMP3410) {
                dev_err(dev, "%s - invalid buffer signature\n", __func__);
                status = -ENODEV;
                goto out;
        }

        do {
                /* Validate the I2C */
                status = read_rom(serial,
                                start_address,
                                sizeof(struct ti_i2c_desc),
                                (u8 *)rom_desc);
                if (status)
                        break;

                if ((start_address + sizeof(struct ti_i2c_desc) +
                        le16_to_cpu(rom_desc->Size)) > TI_MAX_I2C_SIZE) {
                        status = -ENODEV;
                        dev_dbg(dev, "%s - structure too big, erroring out.\n", __func__);
                        break;
                }

                dev_dbg(dev, "%s Type = 0x%x\n", __func__, rom_desc->Type);

                /* Skip type 2 record */
                ttype = rom_desc->Type & 0x0f;
                if (ttype != I2C_DESC_TYPE_FIRMWARE_BASIC
                        && ttype != I2C_DESC_TYPE_FIRMWARE_AUTO) {
                        /* Read the descriptor data */
                        status = read_rom(serial, start_address +
                                                sizeof(struct ti_i2c_desc),
                                                le16_to_cpu(rom_desc->Size),
                                                buffer);
                        if (status)
                                break;

                        status = valid_csum(rom_desc, buffer);
                        if (status)
                                break;
                }
                start_address = start_address + sizeof(struct ti_i2c_desc) +
                                                le16_to_cpu(rom_desc->Size);

        } while ((rom_desc->Type != I2C_DESC_TYPE_ION) &&
                                (start_address < TI_MAX_I2C_SIZE));

        if ((rom_desc->Type != I2C_DESC_TYPE_ION) ||
                                (start_address > TI_MAX_I2C_SIZE))
                status = -ENODEV;

out:
        kfree(buffer);
        kfree(rom_desc);
        return status;
}

static int get_manuf_info(struct edgeport_serial *serial, u8 *buffer)
{
        int status;
        int start_address;
        struct ti_i2c_desc *rom_desc;
        struct edge_ti_manuf_descriptor *desc;
        struct device *dev = &serial->serial->dev->dev;

        rom_desc = kmalloc(sizeof(*rom_desc), GFP_KERNEL);
        if (!rom_desc)
                return -ENOMEM;

        start_address = get_descriptor_addr(serial, I2C_DESC_TYPE_ION,
                                                                rom_desc);

        if (!start_address) {
                dev_dbg(dev, "%s - Edge Descriptor not found in I2C\n", __func__);
                status = -ENODEV;
                goto exit;
        }

        /* Read the descriptor data */
        status = read_rom(serial, start_address+sizeof(struct ti_i2c_desc),
                                        le16_to_cpu(rom_desc->Size), buffer);
        if (status)
                goto exit;

        status = valid_csum(rom_desc, buffer);

        desc = (struct edge_ti_manuf_descriptor *)buffer;
        dev_dbg(dev, "%s - IonConfig      0x%x\n", __func__, desc->IonConfig);
        dev_dbg(dev, "%s - Version          %d\n", __func__, desc->Version);
        dev_dbg(dev, "%s - Cpu/Board      0x%x\n", __func__, desc->CpuRev_BoardRev);
        dev_dbg(dev, "%s - NumPorts         %d\n", __func__, desc->NumPorts);
        dev_dbg(dev, "%s - NumVirtualPorts  %d\n", __func__, desc->NumVirtualPorts);
        dev_dbg(dev, "%s - TotalPorts       %d\n", __func__, desc->TotalPorts);

exit:
        kfree(rom_desc);
        return status;
}

/* Build firmware header used for firmware update */
static int build_i2c_fw_hdr(u8 *header, const struct firmware *fw)
{
        u8 *buffer;
        int buffer_size;
        int i;
        u8 cs = 0;
        struct ti_i2c_desc *i2c_header;
        struct ti_i2c_image_header *img_header;
        struct ti_i2c_firmware_rec *firmware_rec;
        struct edgeport_fw_hdr *fw_hdr = (struct edgeport_fw_hdr *)fw->data;

        /*
         * In order to update the I2C firmware we must change the type 2 record
         * to type 0xF2.  This will force the UMP to come up in Boot Mode.
         * Then while in boot mode, the driver will download the latest
         * firmware (padded to 15.5k) into the UMP ram.  And finally when the
         * device comes back up in download mode the driver will cause the new
         * firmware to be copied from the UMP Ram to I2C and the firmware will
         * update the record type from 0xf2 to 0x02.
         */

        /*
         * Allocate a 15.5k buffer + 2 bytes for version number (Firmware
         * Record)
         */
        buffer_size = (((1024 * 16) - 512 ) +
                        sizeof(struct ti_i2c_firmware_rec));

        buffer = kmalloc(buffer_size, GFP_KERNEL);
        if (!buffer)
                return -ENOMEM;

        /* Set entire image of 0xffs */
        memset(buffer, 0xff, buffer_size);

        /* Copy version number into firmware record */
        firmware_rec = (struct ti_i2c_firmware_rec *)buffer;

        firmware_rec->Ver_Major        = fw_hdr->major_version;
        firmware_rec->Ver_Minor        = fw_hdr->minor_version;

        /* Pointer to fw_down memory image */
        img_header = (struct ti_i2c_image_header *)&fw->data[4];

        memcpy(buffer + sizeof(struct ti_i2c_firmware_rec),
                &fw->data[4 + sizeof(struct ti_i2c_image_header)],
                le16_to_cpu(img_header->Length));

        for (i=0; i < buffer_size; i++) {
                cs = (u8)(cs + buffer[i]);
        }

        kfree(buffer);

        /* Build new header */
        i2c_header =  (struct ti_i2c_desc *)header;
        firmware_rec =  (struct ti_i2c_firmware_rec*)i2c_header->Data;

        i2c_header->Type        = I2C_DESC_TYPE_FIRMWARE_BLANK;
        i2c_header->Size        = cpu_to_le16(buffer_size);
        i2c_header->CheckSum        = cs;
        firmware_rec->Ver_Major        = fw_hdr->major_version;
        firmware_rec->Ver_Minor        = fw_hdr->minor_version;

        return 0;
}

/* Try to figure out what type of I2c we have */
static int i2c_type_bootmode(struct edgeport_serial *serial)
{
        struct device *dev = &serial->serial->dev->dev;
        int status;
        u8 *data;

        data = kmalloc(1, GFP_KERNEL);
        if (!data)
                return -ENOMEM;

        /* Try to read type 2 */
        status = ti_vread_sync(serial->serial->dev, UMPC_MEMORY_READ,
                                DTK_ADDR_SPACE_I2C_TYPE_II, 0, data, 0x01);
        if (status)
                dev_dbg(dev, "%s - read 2 status error = %d\n", __func__, status);
        else
                dev_dbg(dev, "%s - read 2 data = 0x%x\n", __func__, *data);
        if ((!status) && (*data == UMP5152 || *data == UMP3410)) {
                dev_dbg(dev, "%s - ROM_TYPE_II\n", __func__);
                serial->TI_I2C_Type = DTK_ADDR_SPACE_I2C_TYPE_II;
                goto out;
        }

        /* Try to read type 3 */
        status = ti_vread_sync(serial->serial->dev, UMPC_MEMORY_READ,
                                DTK_ADDR_SPACE_I2C_TYPE_III, 0,        data, 0x01);
        if (status)
                dev_dbg(dev, "%s - read 3 status error = %d\n", __func__, status);
        else
                dev_dbg(dev, "%s - read 2 data = 0x%x\n", __func__, *data);
        if ((!status) && (*data == UMP5152 || *data == UMP3410)) {
                dev_dbg(dev, "%s - ROM_TYPE_III\n", __func__);
                serial->TI_I2C_Type = DTK_ADDR_SPACE_I2C_TYPE_III;
                goto out;
        }

        dev_dbg(dev, "%s - Unknown\n", __func__);
        serial->TI_I2C_Type = DTK_ADDR_SPACE_I2C_TYPE_II;
        status = -ENODEV;
out:
        kfree(data);
        return status;
}

static int bulk_xfer(struct usb_serial *serial, void *buffer,
                                                int length, int *num_sent)
{
        int status;

        status = usb_bulk_msg(serial->dev,
                        usb_sndbulkpipe(serial->dev,
                                serial->port[0]->bulk_out_endpointAddress),
                        buffer, length, num_sent, 1000);
        return status;
}

/* Download given firmware image to the device (IN BOOT MODE) */
static int download_code(struct edgeport_serial *serial, u8 *image,
                                                        int image_length)
{
        int status = 0;
        int pos;
        int transfer;
        int done;

        /* Transfer firmware image */
        for (pos = 0; pos < image_length; ) {
                /* Read the next buffer from file */
                transfer = image_length - pos;
                if (transfer > EDGE_FW_BULK_MAX_PACKET_SIZE)
                        transfer = EDGE_FW_BULK_MAX_PACKET_SIZE;

                /* Transfer data */
                status = bulk_xfer(serial->serial, &image[pos],
                                                        transfer, &done);
                if (status)
                        break;
                /* Advance buffer pointer */
                pos += done;
        }

        return status;
}

/* FIXME!!! */
static int config_boot_dev(struct usb_device *dev)
{
        return 0;
}

static int ti_cpu_rev(struct edge_ti_manuf_descriptor *desc)
{
        return TI_GET_CPU_REVISION(desc->CpuRev_BoardRev);
}

static int check_fw_sanity(struct edgeport_serial *serial,
                const struct firmware *fw)
{
        u16 length_total;
        u8 checksum = 0;
        int pos;
        struct device *dev = &serial->serial->interface->dev;
        struct edgeport_fw_hdr *fw_hdr = (struct edgeport_fw_hdr *)fw->data;

        if (fw->size < sizeof(struct edgeport_fw_hdr)) {
                dev_err(dev, "incomplete fw header\n");
                return -EINVAL;
        }

        length_total = le16_to_cpu(fw_hdr->length) +
                        sizeof(struct edgeport_fw_hdr);

        if (fw->size != length_total) {
                dev_err(dev, "bad fw size (expected: %u, got: %zu)\n",
                                length_total, fw->size);
                return -EINVAL;
        }

        for (pos = sizeof(struct edgeport_fw_hdr); pos < fw->size; ++pos)
                checksum += fw->data[pos];

        if (checksum != fw_hdr->checksum) {
                dev_err(dev, "bad fw checksum (expected: 0x%x, got: 0x%x)\n",
                                fw_hdr->checksum, checksum);
                return -EINVAL;
        }

        return 0;
}

/*
 * DownloadTIFirmware - Download run-time operating firmware to the TI5052
 *
 * This routine downloads the main operating code into the TI5052, using the
 * boot code already burned into E2PROM or ROM.
 */
static int download_fw(struct edgeport_serial *serial)
{
        struct device *dev = &serial->serial->interface->dev;
        int status = 0;
        struct usb_interface_descriptor *interface;
        const struct firmware *fw;
        const char *fw_name = "edgeport/down3.bin";
        struct edgeport_fw_hdr *fw_hdr;

        status = request_firmware(&fw, fw_name, dev);
        if (status) {
                dev_err(dev, "Failed to load image \"%s\" err %d\n",
                                fw_name, status);
                return status;
        }

        if (check_fw_sanity(serial, fw)) {
                status = -EINVAL;
                goto out;
        }

        fw_hdr = (struct edgeport_fw_hdr *)fw->data;

        /* If on-board version is newer, "fw_version" will be updated later. */
        serial->fw_version = (fw_hdr->major_version << 8) +
                        fw_hdr->minor_version;

        /*
         * This routine is entered by both the BOOT mode and the Download mode
         * We can determine which code is running by the reading the config
         * descriptor and if we have only one bulk pipe it is in boot mode
         */
        serial->product_info.hardware_type = HARDWARE_TYPE_TIUMP;

        /* Default to type 2 i2c */
        serial->TI_I2C_Type = DTK_ADDR_SPACE_I2C_TYPE_II;

        status = choose_config(serial->serial->dev);
        if (status)
                goto out;

        interface = &serial->serial->interface->cur_altsetting->desc;
        if (!interface) {
                dev_err(dev, "%s - no interface set, error!\n", __func__);
                status = -ENODEV;
                goto out;
        }

        /*
         * Setup initial mode -- the default mode 0 is TI_MODE_CONFIGURING
         * if we have more than one endpoint we are definitely in download
         * mode
         */
        if (interface->bNumEndpoints > 1) {
                serial->product_info.TiMode = TI_MODE_DOWNLOAD;
                status = do_download_mode(serial, fw);
        } else {
                /* Otherwise we will remain in configuring mode */
                serial->product_info.TiMode = TI_MODE_CONFIGURING;
                status = do_boot_mode(serial, fw);
        }

out:
        release_firmware(fw);
        return status;
}

static int do_download_mode(struct edgeport_serial *serial,
                const struct firmware *fw)
{
        struct device *dev = &serial->serial->interface->dev;
        int status = 0;
        int start_address;
        struct edge_ti_manuf_descriptor *ti_manuf_desc;
        int download_cur_ver;
        int download_new_ver;
        struct edgeport_fw_hdr *fw_hdr = (struct edgeport_fw_hdr *)fw->data;
        struct ti_i2c_desc *rom_desc;

        dev_dbg(dev, "%s - RUNNING IN DOWNLOAD MODE\n", __func__);

        status = check_i2c_image(serial);
        if (status) {
                dev_dbg(dev, "%s - DOWNLOAD MODE -- BAD I2C\n", __func__);
                return status;
        }

        /*
         * Validate Hardware version number
         * Read Manufacturing Descriptor from TI Based Edgeport
         */
        ti_manuf_desc = kmalloc(sizeof(*ti_manuf_desc), GFP_KERNEL);
        if (!ti_manuf_desc)
                return -ENOMEM;

        status = get_manuf_info(serial, (u8 *)ti_manuf_desc);
        if (status) {
                kfree(ti_manuf_desc);
                return status;
        }

        /* Check version number of ION descriptor */
        if (!ignore_cpu_rev && ti_cpu_rev(ti_manuf_desc) < 2) {
                dev_dbg(dev, "%s - Wrong CPU Rev %d (Must be 2)\n",
                        __func__, ti_cpu_rev(ti_manuf_desc));
                kfree(ti_manuf_desc);
                return -EINVAL;
        }

        rom_desc = kmalloc(sizeof(*rom_desc), GFP_KERNEL);
        if (!rom_desc) {
                kfree(ti_manuf_desc);
                return -ENOMEM;
        }

        /* Search for type 2 record (firmware record) */
        start_address = get_descriptor_addr(serial,
                        I2C_DESC_TYPE_FIRMWARE_BASIC, rom_desc);
        if (start_address != 0) {
                struct ti_i2c_firmware_rec *firmware_version;
                u8 *record;

                dev_dbg(dev, "%s - Found Type FIRMWARE (Type 2) record\n",
                                __func__);

                firmware_version = kmalloc(sizeof(*firmware_version),
                                                        GFP_KERNEL);
                if (!firmware_version) {
                        kfree(rom_desc);
                        kfree(ti_manuf_desc);
                        return -ENOMEM;
                }

                /*
                 * Validate version number
                 * Read the descriptor data
                 */
                status = read_rom(serial, start_address +
                                sizeof(struct ti_i2c_desc),
                                sizeof(struct ti_i2c_firmware_rec),
                                (u8 *)firmware_version);
                if (status) {
                        kfree(firmware_version);
                        kfree(rom_desc);
                        kfree(ti_manuf_desc);
                        return status;
                }

                /*
                 * Check version number of download with current
                 * version in I2c
                 */
                download_cur_ver = (firmware_version->Ver_Major << 8) +
                                   (firmware_version->Ver_Minor);
                download_new_ver = (fw_hdr->major_version << 8) +
                                   (fw_hdr->minor_version);

                dev_dbg(dev, "%s - >> FW Versions Device %d.%d  Driver %d.%d\n",
                        __func__, firmware_version->Ver_Major,
                        firmware_version->Ver_Minor,
                        fw_hdr->major_version, fw_hdr->minor_version);

                /*
                 * Check if we have an old version in the I2C and
                 * update if necessary
                 */
                if (download_cur_ver < download_new_ver) {
                        dev_dbg(dev, "%s - Update I2C dld from %d.%d to %d.%d\n",
                                __func__,
                                firmware_version->Ver_Major,
                                firmware_version->Ver_Minor,
                                fw_hdr->major_version,
                                fw_hdr->minor_version);

                        record = kmalloc(1, GFP_KERNEL);
                        if (!record) {
                                kfree(firmware_version);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return -ENOMEM;
                        }
                        /*
                         * In order to update the I2C firmware we must
                         * change the type 2 record to type 0xF2. This
                         * will force the UMP to come up in Boot Mode.
                         * Then while in boot mode, the driver will
                         * download the latest firmware (padded to
                         * 15.5k) into the UMP ram. Finally when the
                         * device comes back up in download mode the
                         * driver will cause the new firmware to be
                         * copied from the UMP Ram to I2C and the
                         * firmware will update the record type from
                         * 0xf2 to 0x02.
                         */
                        *record = I2C_DESC_TYPE_FIRMWARE_BLANK;

                        /*
                         * Change the I2C Firmware record type to
                         * 0xf2 to trigger an update
                         */
                        status = write_rom(serial, start_address,
                                        sizeof(*record), record);
                        if (status) {
                                kfree(record);
                                kfree(firmware_version);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return status;
                        }

                        /*
                         * verify the write -- must do this in order
                         * for write to complete before we do the
                         * hardware reset
                         */
                        status = read_rom(serial,
                                                start_address,
                                                sizeof(*record),
                                                record);
                        if (status) {
                                kfree(record);
                                kfree(firmware_version);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return status;
                        }

                        if (*record != I2C_DESC_TYPE_FIRMWARE_BLANK) {
                                dev_err(dev, "%s - error resetting device\n",
                                                __func__);
                                kfree(record);
                                kfree(firmware_version);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return -ENODEV;
                        }

                        dev_dbg(dev, "%s - HARDWARE RESET\n", __func__);

                        /* Reset UMP -- Back to BOOT MODE */
                        status = ti_vsend_sync(serial->serial->dev,
                                        UMPC_HARDWARE_RESET,
                                        0, 0, NULL, 0,
                                        TI_VSEND_TIMEOUT_DEFAULT);

                        dev_dbg(dev, "%s - HARDWARE RESET return %d\n",
                                        __func__, status);

                        /* return an error on purpose. */
                        kfree(record);
                        kfree(firmware_version);
                        kfree(rom_desc);
                        kfree(ti_manuf_desc);
                        return -ENODEV;
                }
                /* Same or newer fw version is already loaded */
                serial->fw_version = download_cur_ver;
                kfree(firmware_version);
        }
        /* Search for type 0xF2 record (firmware blank record) */
        else {
                start_address = get_descriptor_addr(serial,
                                I2C_DESC_TYPE_FIRMWARE_BLANK, rom_desc);
                if (start_address != 0) {
#define HEADER_SIZE        (sizeof(struct ti_i2c_desc) + \
                                sizeof(struct ti_i2c_firmware_rec))
                        u8 *header;
                        u8 *vheader;

                        header = kmalloc(HEADER_SIZE, GFP_KERNEL);
                        if (!header) {
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return -ENOMEM;
                        }

                        vheader = kmalloc(HEADER_SIZE, GFP_KERNEL);
                        if (!vheader) {
                                kfree(header);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return -ENOMEM;
                        }

                        dev_dbg(dev, "%s - Found Type BLANK FIRMWARE (Type F2) record\n",
                                        __func__);

                        /*
                         * In order to update the I2C firmware we must change
                         * the type 2 record to type 0xF2. This will force the
                         * UMP to come up in Boot Mode.  Then while in boot
                         * mode, the driver will download the latest firmware
                         * (padded to 15.5k) into the UMP ram. Finally when the
                         * device comes back up in download mode the driver
                         * will cause the new firmware to be copied from the
                         * UMP Ram to I2C and the firmware will update the
                         * record type from 0xf2 to 0x02.
                         */
                        status = build_i2c_fw_hdr(header, fw);
                        if (status) {
                                kfree(vheader);
                                kfree(header);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return -EINVAL;
                        }

                        /*
                         * Update I2C with type 0xf2 record with correct
                         * size and checksum
                         */
                        status = write_rom(serial,
                                                start_address,
                                                HEADER_SIZE,
                                                header);
                        if (status) {
                                kfree(vheader);
                                kfree(header);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return -EINVAL;
                        }

                        /*
                         * verify the write -- must do this in order for
                         * write to complete before we do the hardware reset
                         */
                        status = read_rom(serial, start_address,
                                                        HEADER_SIZE, vheader);

                        if (status) {
                                dev_dbg(dev, "%s - can't read header back\n",
                                                __func__);
                                kfree(vheader);
                                kfree(header);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return status;
                        }
                        if (memcmp(vheader, header, HEADER_SIZE)) {
                                dev_dbg(dev, "%s - write download record failed\n",
                                                __func__);
                                kfree(vheader);
                                kfree(header);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return -EINVAL;
                        }

                        kfree(vheader);
                        kfree(header);

                        dev_dbg(dev, "%s - Start firmware update\n", __func__);

                        /* Tell firmware to copy download image into I2C */
                        status = ti_vsend_sync(serial->serial->dev,
                                        UMPC_COPY_DNLD_TO_I2C,
                                        0, 0, NULL, 0,
                                        TI_VSEND_TIMEOUT_FW_DOWNLOAD);

                        dev_dbg(dev, "%s - Update complete 0x%x\n", __func__,
                                        status);
                        if (status) {
                                dev_err(dev,
                                        "%s - UMPC_COPY_DNLD_TO_I2C failed\n",
                                        __func__);
                                kfree(rom_desc);
                                kfree(ti_manuf_desc);
                                return status;
                        }
                }
        }

        /* The device is running the download code */
        kfree(rom_desc);
        kfree(ti_manuf_desc);
        return 0;
}

static int do_boot_mode(struct edgeport_serial *serial,
                const struct firmware *fw)
{
        struct device *dev = &serial->serial->interface->dev;
        int status = 0;
        struct edge_ti_manuf_descriptor *ti_manuf_desc;
        struct edgeport_fw_hdr *fw_hdr = (struct edgeport_fw_hdr *)fw->data;

        dev_dbg(dev, "%s - RUNNING IN BOOT MODE\n", __func__);

        /* Configure the TI device so we can use the BULK pipes for download */
        status = config_boot_dev(serial->serial->dev);
        if (status)
                return status;

        if (le16_to_cpu(serial->serial->dev->descriptor.idVendor)
                                                        != USB_VENDOR_ID_ION) {
                dev_dbg(dev, "%s - VID = 0x%x\n", __func__,
                        le16_to_cpu(serial->serial->dev->descriptor.idVendor));
                serial->TI_I2C_Type = DTK_ADDR_SPACE_I2C_TYPE_II;
                goto stayinbootmode;
        }

        /*
         * We have an ION device (I2c Must be programmed)
         * Determine I2C image type
         */
        if (i2c_type_bootmode(serial))
                goto stayinbootmode;

        /* Check for ION Vendor ID and that the I2C is valid */
        if (!check_i2c_image(serial)) {
                struct ti_i2c_image_header *header;
                int i;
                u8 cs = 0;
                u8 *buffer;
                int buffer_size;

                /*
                 * Validate Hardware version number
                 * Read Manufacturing Descriptor from TI Based Edgeport
                 */
                ti_manuf_desc = kmalloc(sizeof(*ti_manuf_desc), GFP_KERNEL);
                if (!ti_manuf_desc)
                        return -ENOMEM;

                status = get_manuf_info(serial, (u8 *)ti_manuf_desc);
                if (status) {
                        kfree(ti_manuf_desc);
                        goto stayinbootmode;
                }

                /* Check for version 2 */
                if (!ignore_cpu_rev && ti_cpu_rev(ti_manuf_desc) < 2) {
                        dev_dbg(dev, "%s - Wrong CPU Rev %d (Must be 2)\n",
                                __func__, ti_cpu_rev(ti_manuf_desc));
                        kfree(ti_manuf_desc);
                        goto stayinbootmode;
                }

                kfree(ti_manuf_desc);

                /*
                 * In order to update the I2C firmware we must change the type
                 * 2 record to type 0xF2. This will force the UMP to come up
                 * in Boot Mode.  Then while in boot mode, the driver will
                 * download the latest firmware (padded to 15.5k) into the
                 * UMP ram. Finally when the device comes back up in download
                 * mode the driver will cause the new firmware to be copied
                 * from the UMP Ram to I2C and the firmware will update the
                 * record type from 0xf2 to 0x02.
                 *
                 * Do we really have to copy the whole firmware image,
                 * or could we do this in place!
                 */

                /* Allocate a 15.5k buffer + 3 byte header */
                buffer_size = (((1024 * 16) - 512) +
                                        sizeof(struct ti_i2c_image_header));
                buffer = kmalloc(buffer_size, GFP_KERNEL);
                if (!buffer)
                        return -ENOMEM;

                /* Initialize the buffer to 0xff (pad the buffer) */
                memset(buffer, 0xff, buffer_size);
                memcpy(buffer, &fw->data[4], fw->size - 4);

                for (i = sizeof(struct ti_i2c_image_header);
                                i < buffer_size; i++) {
                        cs = (u8)(cs + buffer[i]);
                }

                header = (struct ti_i2c_image_header *)buffer;

                /* update length and checksum after padding */
                header->Length = cpu_to_le16((u16)(buffer_size -
                                        sizeof(struct ti_i2c_image_header)));
                header->CheckSum = cs;

                /* Download the operational code  */
                dev_dbg(dev, "%s - Downloading operational code image version %d.%d (TI UMP)\n",
                                __func__,
                                fw_hdr->major_version, fw_hdr->minor_version);
                status = download_code(serial, buffer, buffer_size);

                kfree(buffer);

                if (status) {
                        dev_dbg(dev, "%s - Error downloading operational code image\n", __func__);
                        return status;
                }

                /* Device will reboot */
                serial->product_info.TiMode = TI_MODE_TRANSITIONING;

                dev_dbg(dev, "%s - Download successful -- Device rebooting...\n", __func__);

                return 1;
        }

stayinbootmode:
        /* Eprom is invalid or blank stay in boot mode */
        dev_dbg(dev, "%s - STAYING IN BOOT MODE\n", __func__);
        serial->product_info.TiMode = TI_MODE_BOOT;

        return 1;
}

static int ti_do_config(struct edgeport_port *port, int feature, int on)
{
        on = !!on;        /* 1 or 0 not bitmask */

        return send_port_cmd(port->port, feature, on, NULL, 0);
}

static int restore_mcr(struct edgeport_port *port, u8 mcr)
{
        int status = 0;

        dev_dbg(&port->port->dev, "%s - %x\n", __func__, mcr);

        status = ti_do_config(port, UMPC_SET_CLR_DTR, mcr & MCR_DTR);
        if (status)
                return status;
        status = ti_do_config(port, UMPC_SET_CLR_RTS, mcr & MCR_RTS);
        if (status)
                return status;
        return ti_do_config(port, UMPC_SET_CLR_LOOPBACK, mcr & MCR_LOOPBACK);
}

/* Convert TI LSR to standard UART flags */
static u8 map_line_status(u8 ti_lsr)
{
        u8 lsr = 0;

#define MAP_FLAG(flagUmp, flagUart)    \
        if (ti_lsr & flagUmp) \
                lsr |= flagUart;

        MAP_FLAG(UMP_UART_LSR_OV_MASK, LSR_OVER_ERR)        /* overrun */
        MAP_FLAG(UMP_UART_LSR_PE_MASK, LSR_PAR_ERR)        /* parity error */
        MAP_FLAG(UMP_UART_LSR_FE_MASK, LSR_FRM_ERR)        /* framing error */
        MAP_FLAG(UMP_UART_LSR_BR_MASK, LSR_BREAK)        /* break detected */
        MAP_FLAG(UMP_UART_LSR_RX_MASK, LSR_RX_AVAIL)        /* rx data available */
        MAP_FLAG(UMP_UART_LSR_TX_MASK, LSR_TX_EMPTY)        /* tx hold reg empty */

#undef MAP_FLAG

        return lsr;
}

static void handle_new_msr(struct edgeport_port *edge_port, u8 msr)
{
        struct async_icount *icount;
        struct tty_struct *tty;

        dev_dbg(&edge_port->port->dev, "%s - %02x\n", __func__, msr);

        if (msr & (EDGEPORT_MSR_DELTA_CTS | EDGEPORT_MSR_DELTA_DSR |
                        EDGEPORT_MSR_DELTA_RI | EDGEPORT_MSR_DELTA_CD)) {
                icount = &edge_port->port->icount;

                /* update input line counters */
                if (msr & EDGEPORT_MSR_DELTA_CTS)
                        icount->cts++;
                if (msr & EDGEPORT_MSR_DELTA_DSR)
                        icount->dsr++;
                if (msr & EDGEPORT_MSR_DELTA_CD)
                        icount->dcd++;
                if (msr & EDGEPORT_MSR_DELTA_RI)
                        icount->rng++;
                wake_up_interruptible(&edge_port->port->port.delta_msr_wait);
        }

        /* Save the new modem status */
        edge_port->shadow_msr = msr & 0xf0;

        tty = tty_port_tty_get(&edge_port->port->port);
        /* handle CTS flow control */
        if (tty && C_CRTSCTS(tty)) {
                if (msr & EDGEPORT_MSR_CTS)
                        tty_wakeup(tty);
        }
        tty_kref_put(tty);
}

static void handle_new_lsr(struct edgeport_port *edge_port, int lsr_data,
                                                        u8 lsr, u8 data)
{
        struct async_icount *icount;
        u8 new_lsr = (u8)(lsr & (u8)(LSR_OVER_ERR | LSR_PAR_ERR |
                                                LSR_FRM_ERR | LSR_BREAK));

        dev_dbg(&edge_port->port->dev, "%s - %02x\n", __func__, new_lsr);

        edge_port->shadow_lsr = lsr;

        if (new_lsr & LSR_BREAK)
                /*
                 * Parity and Framing errors only count if they
                 * occur exclusive of a break being received.
                 */
                new_lsr &= (u8)(LSR_OVER_ERR | LSR_BREAK);

        /* Place LSR data byte into Rx buffer */
        if (lsr_data)
                edge_tty_recv(edge_port->port, &data, 1);

        /* update input line counters */
        icount = &edge_port->port->icount;
        if (new_lsr & LSR_BREAK)
                icount->brk++;
        if (new_lsr & LSR_OVER_ERR)
                icount->overrun++;
        if (new_lsr & LSR_PAR_ERR)
                icount->parity++;
        if (new_lsr & LSR_FRM_ERR)
                icount->frame++;
}

static void edge_interrupt_callback(struct urb *urb)
{
        struct edgeport_serial *edge_serial = urb->context;
        struct usb_serial_port *port;
        struct edgeport_port *edge_port;
        struct device *dev;
        unsigned char *data = urb->transfer_buffer;
        int length = urb->actual_length;
        int port_number;
        int function;
        int retval;
        u8 lsr;
        u8 msr;
        int status = urb->status;

        switch (status) {
        case 0:
                /* success */
                break;
        case -ECONNRESET:
        case -ENOENT:
        case -ESHUTDOWN:
                /* this urb is terminated, clean up */
                dev_dbg(&urb->dev->dev, "%s - urb shutting down with status: %d\n",
                    __func__, status);
                return;
        default:
                dev_err(&urb->dev->dev, "%s - nonzero urb status received: "
                        "%d\n", __func__, status);
                goto exit;
        }

        if (!length) {
                dev_dbg(&urb->dev->dev, "%s - no data in urb\n", __func__);
                goto exit;
        }

        dev = &edge_serial->serial->dev->dev;
        usb_serial_debug_data(dev, __func__, length, data);

        if (length != 2) {
                dev_dbg(dev, "%s - expecting packet of size 2, got %d\n", __func__, length);
                goto exit;
        }

        port_number = TIUMP_GET_PORT_FROM_CODE(data[0]);
        function    = TIUMP_GET_FUNC_FROM_CODE(data[0]);
        dev_dbg(dev, "%s - port_number %d, function %d, info 0x%x\n", __func__,
                port_number, function, data[1]);

        if (port_number >= edge_serial->serial->num_ports) {
                dev_err(dev, "bad port number %d\n", port_number);
                goto exit;
        }

        port = edge_serial->serial->port[port_number];
        edge_port = usb_get_serial_port_data(port);
        if (!edge_port) {
                dev_dbg(dev, "%s - edge_port not found\n", __func__);
                return;
        }
        switch (function) {
        case TIUMP_INTERRUPT_CODE_LSR:
                lsr = map_line_status(data[1]);
                if (lsr & UMP_UART_LSR_DATA_MASK) {
                        /*
                         * Save the LSR event for bulk read completion routine
                         */
                        dev_dbg(dev, "%s - LSR Event Port %u LSR Status = %02x\n",
                                __func__, port_number, lsr);
                        edge_port->lsr_event = 1;
                        edge_port->lsr_mask = lsr;
                } else {
                        dev_dbg(dev, "%s - ===== Port %d LSR Status = %02x ======\n",
                                __func__, port_number, lsr);
                        handle_new_lsr(edge_port, 0, lsr, 0);
                }
                break;

        case TIUMP_INTERRUPT_CODE_MSR:        /* MSR */
                /* Copy MSR from UMP */
                msr = data[1];
                dev_dbg(dev, "%s - ===== Port %u MSR Status = %02x ======\n",
                        __func__, port_number, msr);
                handle_new_msr(edge_port, msr);
                break;

        default:
                dev_err(&urb->dev->dev,
                        "%s - Unknown Interrupt code from UMP %x\n",
                        __func__, data[1]);
                break;

        }

exit:
        retval = usb_submit_urb(urb, GFP_ATOMIC);
        if (retval)
                dev_err(&urb->dev->dev,
                        "%s - usb_submit_urb failed with result %d\n",
                         __func__, retval);
}

static void edge_bulk_in_callback(struct urb *urb)
{
        struct edgeport_port *edge_port = urb->context;
        struct device *dev = &edge_port->port->dev;
        unsigned char *data = urb->transfer_buffer;
        unsigned long flags;
        int retval = 0;
        int port_number;
        int status = urb->status;

        switch (status) {
        case 0:
                /* success */
                break;
        case -ECONNRESET:
        case -ENOENT:
        case -ESHUTDOWN:
                /* this urb is terminated, clean up */
                dev_dbg(&urb->dev->dev, "%s - urb shutting down with status: %d\n", __func__, status);
                return;
        default:
                dev_err(&urb->dev->dev, "%s - nonzero read bulk status received: %d\n", __func__, status);
        }

        if (status == -EPIPE)
                goto exit;

        if (status) {
                dev_err(&urb->dev->dev, "%s - stopping read!\n", __func__);
                return;
        }

        port_number = edge_port->port->port_number;

        if (urb->actual_length > 0 && edge_port->lsr_event) {
                edge_port->lsr_event = 0;
                dev_dbg(dev, "%s ===== Port %u LSR Status = %02x, Data = %02x ======\n",
                        __func__, port_number, edge_port->lsr_mask, *data);
                handle_new_lsr(edge_port, 1, edge_port->lsr_mask, *data);
                /* Adjust buffer length/pointer */
                --urb->actual_length;
                ++data;
        }

        if (urb->actual_length) {
                usb_serial_debug_data(dev, __func__, urb->actual_length, data);
                if (edge_port->close_pending)
                        dev_dbg(dev, "%s - close pending, dropping data on the floor\n",
                                                                __func__);
                else
                        edge_tty_recv(edge_port->port, data,
                                        urb->actual_length);
                edge_port->port->icount.rx += urb->actual_length;
        }

exit:
        /* continue read unless stopped */
        spin_lock_irqsave(&edge_port->ep_lock, flags);
        if (edge_port->ep_read_urb_state == EDGE_READ_URB_RUNNING)
                retval = usb_submit_urb(urb, GFP_ATOMIC);
        else if (edge_port->ep_read_urb_state == EDGE_READ_URB_STOPPING)
                edge_port->ep_read_urb_state = EDGE_READ_URB_STOPPED;

        spin_unlock_irqrestore(&edge_port->ep_lock, flags);
        if (retval)
                dev_err(dev, "%s - usb_submit_urb failed with result %d\n", __func__, retval);
}

static void edge_tty_recv(struct usb_serial_port *port, unsigned char *data,
                int length)
{
        int queued;

        queued = tty_insert_flip_string(&port->port, data, length);
        if (queued < length)
                dev_err(&port->dev, "%s - dropping data, %d bytes lost\n",
                        __func__, length - queued);
        tty_flip_buffer_push(&port->port);
}

static void edge_bulk_out_callback(struct urb *urb)
{
        struct usb_serial_port *port = urb->context;
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        int status = urb->status;
        struct tty_struct *tty;

        edge_port->ep_write_urb_in_use = 0;

        switch (status) {
        case 0:
                /* success */
                break;
        case -ECONNRESET:
        case -ENOENT:
        case -ESHUTDOWN:
                /* this urb is terminated, clean up */
                dev_dbg(&urb->dev->dev, "%s - urb shutting down with status: %d\n",
                    __func__, status);
                return;
        default:
                dev_err_console(port, "%s - nonzero write bulk status "
                        "received: %d\n", __func__, status);
        }

        /* send any buffered data */
        tty = tty_port_tty_get(&port->port);
        edge_send(port, tty);
        tty_kref_put(tty);
}

static int edge_open(struct tty_struct *tty, struct usb_serial_port *port)
{
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        struct edgeport_serial *edge_serial;
        struct usb_device *dev;
        struct urb *urb;
        int status;
        u16 open_settings;
        u8 transaction_timeout;

        if (edge_port == NULL)
                return -ENODEV;

        dev = port->serial->dev;

        /* turn off loopback */
        status = ti_do_config(edge_port, UMPC_SET_CLR_LOOPBACK, 0);
        if (status) {
                dev_err(&port->dev,
                                "%s - cannot send clear loopback command, %d\n",
                        __func__, status);
                return status;
        }

        /* set up the port settings */
        if (tty)
                edge_set_termios(tty, port, &tty->termios);

        /* open up the port */

        /* milliseconds to timeout for DMA transfer */
        transaction_timeout = 2;

        edge_port->ump_read_timeout =
                                max(20, ((transaction_timeout * 3) / 2));

        /* milliseconds to timeout for DMA transfer */
        open_settings = (u8)(UMP_DMA_MODE_CONTINOUS |
                             UMP_PIPE_TRANS_TIMEOUT_ENA |
                             (transaction_timeout << 2));

        dev_dbg(&port->dev, "%s - Sending UMPC_OPEN_PORT\n", __func__);

        /* Tell TI to open and start the port */
        status = send_port_cmd(port, UMPC_OPEN_PORT, open_settings, NULL, 0);
        if (status) {
                dev_err(&port->dev, "%s - cannot send open command, %d\n",
                                                        __func__, status);
                return status;
        }

        /* Start the DMA? */
        status = send_port_cmd(port, UMPC_START_PORT, 0, NULL, 0);
        if (status) {
                dev_err(&port->dev, "%s - cannot send start DMA command, %d\n",
                                                        __func__, status);
                return status;
        }

        /* Clear TX and RX buffers in UMP */
        status = purge_port(port, UMP_PORT_DIR_OUT | UMP_PORT_DIR_IN);
        if (status) {
                dev_err(&port->dev,
                        "%s - cannot send clear buffers command, %d\n",
                        __func__, status);
                return status;
        }

        /* Read Initial MSR */
        status = read_port_cmd(port, UMPC_READ_MSR, 0, &edge_port->shadow_msr, 1);
        if (status) {
                dev_err(&port->dev, "%s - cannot send read MSR command, %d\n",
                                                        __func__, status);
                return status;
        }

        dev_dbg(&port->dev, "ShadowMSR 0x%X\n", edge_port->shadow_msr);

        /* Set Initial MCR */
        edge_port->shadow_mcr = MCR_RTS | MCR_DTR;
        dev_dbg(&port->dev, "ShadowMCR 0x%X\n", edge_port->shadow_mcr);

        edge_serial = edge_port->edge_serial;
        if (mutex_lock_interruptible(&edge_serial->es_lock))
                return -ERESTARTSYS;
        if (edge_serial->num_ports_open == 0) {
                /* we are the first port to open, post the interrupt urb */
                urb = edge_serial->serial->port[0]->interrupt_in_urb;
                urb->context = edge_serial;
                status = usb_submit_urb(urb, GFP_KERNEL);
                if (status) {
                        dev_err(&port->dev,
                                "%s - usb_submit_urb failed with value %d\n",
                                        __func__, status);
                        goto release_es_lock;
                }
        }

        /*
         * reset the data toggle on the bulk endpoints to work around bug in
         * host controllers where things get out of sync some times
         */
        usb_clear_halt(dev, port->write_urb->pipe);
        usb_clear_halt(dev, port->read_urb->pipe);

        /* start up our bulk read urb */
        urb = port->read_urb;
        edge_port->ep_read_urb_state = EDGE_READ_URB_RUNNING;
        urb->context = edge_port;
        status = usb_submit_urb(urb, GFP_KERNEL);
        if (status) {
                dev_err(&port->dev,
                        "%s - read bulk usb_submit_urb failed with value %d\n",
                                __func__, status);
                goto unlink_int_urb;
        }

        ++edge_serial->num_ports_open;

        goto release_es_lock;

unlink_int_urb:
        if (edge_port->edge_serial->num_ports_open == 0)
                usb_kill_urb(port->serial->port[0]->interrupt_in_urb);
release_es_lock:
        mutex_unlock(&edge_serial->es_lock);
        return status;
}

static void edge_close(struct usb_serial_port *port)
{
        struct edgeport_serial *edge_serial;
        struct edgeport_port *edge_port;
        unsigned long flags;

        edge_serial = usb_get_serial_data(port->serial);
        edge_port = usb_get_serial_port_data(port);
        if (edge_serial == NULL || edge_port == NULL)
                return;

        /*
         * The bulkreadcompletion routine will check
         * this flag and dump add read data
         */
        edge_port->close_pending = 1;

        usb_kill_urb(port->read_urb);
        usb_kill_urb(port->write_urb);
        edge_port->ep_write_urb_in_use = 0;
        spin_lock_irqsave(&edge_port->ep_lock, flags);
        kfifo_reset_out(&port->write_fifo);
        spin_unlock_irqrestore(&edge_port->ep_lock, flags);

        dev_dbg(&port->dev, "%s - send umpc_close_port\n", __func__);
        send_port_cmd(port, UMPC_CLOSE_PORT, 0, NULL, 0);

        mutex_lock(&edge_serial->es_lock);
        --edge_port->edge_serial->num_ports_open;
        if (edge_port->edge_serial->num_ports_open <= 0) {
                /* last port is now closed, let's shut down our interrupt urb */
                usb_kill_urb(port->serial->port[0]->interrupt_in_urb);
                edge_port->edge_serial->num_ports_open = 0;
        }
        mutex_unlock(&edge_serial->es_lock);
        edge_port->close_pending = 0;
}

static int edge_write(struct tty_struct *tty, struct usb_serial_port *port,
                                const unsigned char *data, int count)
{
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);

        if (count == 0) {
                dev_dbg(&port->dev, "%s - write request of 0 bytes\n", __func__);
                return 0;
        }

        if (edge_port == NULL)
                return -ENODEV;
        if (edge_port->close_pending == 1)
                return -ENODEV;

        count = kfifo_in_locked(&port->write_fifo, data, count,
                                                        &edge_port->ep_lock);
        edge_send(port, tty);

        return count;
}

static void edge_send(struct usb_serial_port *port, struct tty_struct *tty)
{
        int count, result;
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        unsigned long flags;

        spin_lock_irqsave(&edge_port->ep_lock, flags);

        if (edge_port->ep_write_urb_in_use) {
                spin_unlock_irqrestore(&edge_port->ep_lock, flags);
                return;
        }

        count = kfifo_out(&port->write_fifo,
                                port->write_urb->transfer_buffer,
                                port->bulk_out_size);

        if (count == 0) {
                spin_unlock_irqrestore(&edge_port->ep_lock, flags);
                return;
        }

        edge_port->ep_write_urb_in_use = 1;

        spin_unlock_irqrestore(&edge_port->ep_lock, flags);

        usb_serial_debug_data(&port->dev, __func__, count, port->write_urb->transfer_buffer);

        /* set up our urb */
        port->write_urb->transfer_buffer_length = count;

        /* send the data out the bulk port */
        result = usb_submit_urb(port->write_urb, GFP_ATOMIC);
        if (result) {
                dev_err_console(port,
                        "%s - failed submitting write urb, error %d\n",
                                __func__, result);
                edge_port->ep_write_urb_in_use = 0;
                /* TODO: reschedule edge_send */
        } else
                edge_port->port->icount.tx += count;

        /*
         * wakeup any process waiting for writes to complete
         * there is now more room in the buffer for new writes
         */
        if (tty)
                tty_wakeup(tty);
}

static unsigned int edge_write_room(struct tty_struct *tty)
{
        struct usb_serial_port *port = tty->driver_data;
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        unsigned int room;
        unsigned long flags;

        if (edge_port == NULL)
                return 0;
        if (edge_port->close_pending == 1)
                return 0;

        spin_lock_irqsave(&edge_port->ep_lock, flags);
        room = kfifo_avail(&port->write_fifo);
        spin_unlock_irqrestore(&edge_port->ep_lock, flags);

        dev_dbg(&port->dev, "%s - returns %u\n", __func__, room);
        return room;
}

static unsigned int edge_chars_in_buffer(struct tty_struct *tty)
{
        struct usb_serial_port *port = tty->driver_data;
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        unsigned int chars;
        unsigned long flags;
        if (edge_port == NULL)
                return 0;

        spin_lock_irqsave(&edge_port->ep_lock, flags);
        chars = kfifo_len(&port->write_fifo);
        spin_unlock_irqrestore(&edge_port->ep_lock, flags);

        dev_dbg(&port->dev, "%s - returns %u\n", __func__, chars);
        return chars;
}

static bool edge_tx_empty(struct usb_serial_port *port)
{
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        int ret;

        ret = tx_active(edge_port);
        if (ret > 0)
                return false;

        return true;
}

static void edge_throttle(struct tty_struct *tty)
{
        struct usb_serial_port *port = tty->driver_data;
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        int status;

        if (edge_port == NULL)
                return;

        /* if we are implementing XON/XOFF, send the stop character */
        if (I_IXOFF(tty)) {
                unsigned char stop_char = STOP_CHAR(tty);
                status = edge_write(tty, port, &stop_char, 1);
                if (status <= 0) {
                        dev_err(&port->dev, "%s - failed to write stop character, %d\n", __func__, status);
                }
        }

        /*
         * if we are implementing RTS/CTS, stop reads
         * and the Edgeport will clear the RTS line
         */
        if (C_CRTSCTS(tty))
                stop_read(edge_port);

}

static void edge_unthrottle(struct tty_struct *tty)
{
        struct usb_serial_port *port = tty->driver_data;
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        int status;

        if (edge_port == NULL)
                return;

        /* if we are implementing XON/XOFF, send the start character */
        if (I_IXOFF(tty)) {
                unsigned char start_char = START_CHAR(tty);
                status = edge_write(tty, port, &start_char, 1);
                if (status <= 0) {
                        dev_err(&port->dev, "%s - failed to write start character, %d\n", __func__, status);
                }
        }
        /*
         * if we are implementing RTS/CTS, restart reads
         * are the Edgeport will assert the RTS line
         */
        if (C_CRTSCTS(tty)) {
                status = restart_read(edge_port);
                if (status)
                        dev_err(&port->dev,
                                "%s - read bulk usb_submit_urb failed: %d\n",
                                                        __func__, status);
        }

}

static void stop_read(struct edgeport_port *edge_port)
{
        unsigned long flags;

        spin_lock_irqsave(&edge_port->ep_lock, flags);

        if (edge_port->ep_read_urb_state == EDGE_READ_URB_RUNNING)
                edge_port->ep_read_urb_state = EDGE_READ_URB_STOPPING;
        edge_port->shadow_mcr &= ~MCR_RTS;

        spin_unlock_irqrestore(&edge_port->ep_lock, flags);
}

static int restart_read(struct edgeport_port *edge_port)
{
        struct urb *urb;
        int status = 0;
        unsigned long flags;

        spin_lock_irqsave(&edge_port->ep_lock, flags);

        if (edge_port->ep_read_urb_state == EDGE_READ_URB_STOPPED) {
                urb = edge_port->port->read_urb;
                status = usb_submit_urb(urb, GFP_ATOMIC);
        }
        edge_port->ep_read_urb_state = EDGE_READ_URB_RUNNING;
        edge_port->shadow_mcr |= MCR_RTS;

        spin_unlock_irqrestore(&edge_port->ep_lock, flags);

        return status;
}

static void change_port_settings(struct tty_struct *tty,
                struct edgeport_port *edge_port, const struct ktermios *old_termios)
{
        struct device *dev = &edge_port->port->dev;
        struct ump_uart_config *config;
        int baud;
        unsigned cflag;
        int status;

        config = kmalloc (sizeof (*config), GFP_KERNEL);
        if (!config) {
                tty->termios = *old_termios;
                return;
        }

        cflag = tty->termios.c_cflag;

        config->wFlags = 0;

        /* These flags must be set */
        config->wFlags |= UMP_MASK_UART_FLAGS_RECEIVE_MS_INT;
        config->wFlags |= UMP_MASK_UART_FLAGS_AUTO_START_ON_ERR;
        config->bUartMode = (u8)(edge_port->bUartMode);

        switch (cflag & CSIZE) {
        case CS5:
                    config->bDataBits = UMP_UART_CHAR5BITS;
                    dev_dbg(dev, "%s - data bits = 5\n", __func__);
                    break;
        case CS6:
                    config->bDataBits = UMP_UART_CHAR6BITS;
                    dev_dbg(dev, "%s - data bits = 6\n", __func__);
                    break;
        case CS7:
                    config->bDataBits = UMP_UART_CHAR7BITS;
                    dev_dbg(dev, "%s - data bits = 7\n", __func__);
                    break;
        default:
        case CS8:
                    config->bDataBits = UMP_UART_CHAR8BITS;
                    dev_dbg(dev, "%s - data bits = 8\n", __func__);
                            break;
        }

        if (cflag & PARENB) {
                if (cflag & PARODD) {
                        config->wFlags |= UMP_MASK_UART_FLAGS_PARITY;
                        config->bParity = UMP_UART_ODDPARITY;
                        dev_dbg(dev, "%s - parity = odd\n", __func__);
                } else {
                        config->wFlags |= UMP_MASK_UART_FLAGS_PARITY;
                        config->bParity = UMP_UART_EVENPARITY;
                        dev_dbg(dev, "%s - parity = even\n", __func__);
                }
        } else {
                config->bParity = UMP_UART_NOPARITY;
                dev_dbg(dev, "%s - parity = none\n", __func__);
        }

        if (cflag & CSTOPB) {
                config->bStopBits = UMP_UART_STOPBIT2;
                dev_dbg(dev, "%s - stop bits = 2\n", __func__);
        } else {
                config->bStopBits = UMP_UART_STOPBIT1;
                dev_dbg(dev, "%s - stop bits = 1\n", __func__);
        }

        /* figure out the flow control settings */
        if (cflag & CRTSCTS) {
                config->wFlags |= UMP_MASK_UART_FLAGS_OUT_X_CTS_FLOW;
                config->wFlags |= UMP_MASK_UART_FLAGS_RTS_FLOW;
                dev_dbg(dev, "%s - RTS/CTS is enabled\n", __func__);
        } else {
                dev_dbg(dev, "%s - RTS/CTS is disabled\n", __func__);
                restart_read(edge_port);
        }

        /*
         * if we are implementing XON/XOFF, set the start and stop
         * character in the device
         */
        config->cXon  = START_CHAR(tty);
        config->cXoff = STOP_CHAR(tty);

        /* if we are implementing INBOUND XON/XOFF */
        if (I_IXOFF(tty)) {
                config->wFlags |= UMP_MASK_UART_FLAGS_IN_X;
                dev_dbg(dev, "%s - INBOUND XON/XOFF is enabled, XON = %2x, XOFF = %2x\n",
                        __func__, config->cXon, config->cXoff);
        } else
                dev_dbg(dev, "%s - INBOUND XON/XOFF is disabled\n", __func__);

        /* if we are implementing OUTBOUND XON/XOFF */
        if (I_IXON(tty)) {
                config->wFlags |= UMP_MASK_UART_FLAGS_OUT_X;
                dev_dbg(dev, "%s - OUTBOUND XON/XOFF is enabled, XON = %2x, XOFF = %2x\n",
                        __func__, config->cXon, config->cXoff);
        } else
                dev_dbg(dev, "%s - OUTBOUND XON/XOFF is disabled\n", __func__);

        tty->termios.c_cflag &= ~CMSPAR;

        /* Round the baud rate */
        baud = tty_get_baud_rate(tty);
        if (!baud) {
                /* pick a default, any default... */
                baud = 9600;
        } else {
                /* Avoid a zero divisor. */
                baud = min(baud, 461550);
                tty_encode_baud_rate(tty, baud, baud);
        }

        edge_port->baud_rate = baud;
        config->wBaudRate = (u16)((461550L + baud/2) / baud);

        /* FIXME: Recompute actual baud from divisor here */

        dev_dbg(dev, "%s - baud rate = %d, wBaudRate = %d\n", __func__, baud, config->wBaudRate);

        dev_dbg(dev, "wBaudRate:   %d\n", (int)(461550L / config->wBaudRate));
        dev_dbg(dev, "wFlags:    0x%x\n", config->wFlags);
        dev_dbg(dev, "bDataBits:   %d\n", config->bDataBits);
        dev_dbg(dev, "bParity:     %d\n", config->bParity);
        dev_dbg(dev, "bStopBits:   %d\n", config->bStopBits);
        dev_dbg(dev, "cXon:        %d\n", config->cXon);
        dev_dbg(dev, "cXoff:       %d\n", config->cXoff);
        dev_dbg(dev, "bUartMode:   %d\n", config->bUartMode);

        /* move the word values into big endian mode */
        cpu_to_be16s(&config->wFlags);
        cpu_to_be16s(&config->wBaudRate);

        status = send_port_cmd(edge_port->port, UMPC_SET_CONFIG, 0, config,
                        sizeof(*config));
        if (status)
                dev_dbg(dev, "%s - error %d when trying to write config to device\n",
                        __func__, status);
        kfree(config);
}

static void edge_set_termios(struct tty_struct *tty,
                             struct usb_serial_port *port,
                             const struct ktermios *old_termios)
{
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);

        if (edge_port == NULL)
                return;
        /* change the port settings to the new ones specified */
        change_port_settings(tty, edge_port, old_termios);
}

static int edge_tiocmset(struct tty_struct *tty,
                                        unsigned int set, unsigned int clear)
{
        struct usb_serial_port *port = tty->driver_data;
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        unsigned int mcr;
        unsigned long flags;

        spin_lock_irqsave(&edge_port->ep_lock, flags);
        mcr = edge_port->shadow_mcr;
        if (set & TIOCM_RTS)
                mcr |= MCR_RTS;
        if (set & TIOCM_DTR)
                mcr |= MCR_DTR;
        if (set & TIOCM_LOOP)
                mcr |= MCR_LOOPBACK;

        if (clear & TIOCM_RTS)
                mcr &= ~MCR_RTS;
        if (clear & TIOCM_DTR)
                mcr &= ~MCR_DTR;
        if (clear & TIOCM_LOOP)
                mcr &= ~MCR_LOOPBACK;

        edge_port->shadow_mcr = mcr;
        spin_unlock_irqrestore(&edge_port->ep_lock, flags);

        restore_mcr(edge_port, mcr);
        return 0;
}

static int edge_tiocmget(struct tty_struct *tty)
{
        struct usb_serial_port *port = tty->driver_data;
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        unsigned int result = 0;
        unsigned int msr;
        unsigned int mcr;
        unsigned long flags;

        spin_lock_irqsave(&edge_port->ep_lock, flags);

        msr = edge_port->shadow_msr;
        mcr = edge_port->shadow_mcr;
        result = ((mcr & MCR_DTR)        ? TIOCM_DTR: 0)          /* 0x002 */
                  | ((mcr & MCR_RTS)        ? TIOCM_RTS: 0)   /* 0x004 */
                  | ((msr & EDGEPORT_MSR_CTS)        ? TIOCM_CTS: 0)   /* 0x020 */
                  | ((msr & EDGEPORT_MSR_CD)        ? TIOCM_CAR: 0)   /* 0x040 */
                  | ((msr & EDGEPORT_MSR_RI)        ? TIOCM_RI:  0)   /* 0x080 */
                  | ((msr & EDGEPORT_MSR_DSR)        ? TIOCM_DSR: 0);  /* 0x100 */


        dev_dbg(&port->dev, "%s -- %x\n", __func__, result);
        spin_unlock_irqrestore(&edge_port->ep_lock, flags);

        return result;
}

static int edge_break(struct tty_struct *tty, int break_state)
{
        struct usb_serial_port *port = tty->driver_data;
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        int status;
        int bv = 0;        /* Off */

        if (break_state == -1)
                bv = 1;        /* On */

        status = ti_do_config(edge_port, UMPC_SET_CLR_BREAK, bv);
        if (status) {
                dev_dbg(&port->dev, "%s - error %d sending break set/clear command.\n",
                        __func__, status);
                return status;
        }

        return 0;
}

static void edge_heartbeat_schedule(struct edgeport_serial *edge_serial)
{
        if (!edge_serial->use_heartbeat)
                return;

        schedule_delayed_work(&edge_serial->heartbeat_work,
                        FW_HEARTBEAT_SECS * HZ);
}

static void edge_heartbeat_work(struct work_struct *work)
{
        struct edgeport_serial *serial;
        struct ti_i2c_desc *rom_desc;

        serial = container_of(work, struct edgeport_serial,
                        heartbeat_work.work);

        rom_desc = kmalloc(sizeof(*rom_desc), GFP_KERNEL);

        /* Descriptor address request is enough to reset the firmware timer */
        if (!rom_desc || !get_descriptor_addr(serial, I2C_DESC_TYPE_ION,
                        rom_desc)) {
                dev_err(&serial->serial->interface->dev,
                                "%s - Incomplete heartbeat\n", __func__);
        }
        kfree(rom_desc);

        edge_heartbeat_schedule(serial);
}

static int edge_calc_num_ports(struct usb_serial *serial,
                                struct usb_serial_endpoints *epds)
{
        struct device *dev = &serial->interface->dev;
        unsigned char num_ports = serial->type->num_ports;

        /* Make sure we have the required endpoints when in download mode. */
        if (serial->interface->cur_altsetting->desc.bNumEndpoints > 1) {
                if (epds->num_bulk_in < num_ports ||
                                epds->num_bulk_out < num_ports ||
                                epds->num_interrupt_in < 1) {
                        dev_err(dev, "required endpoints missing\n");
                        return -ENODEV;
                }
        }

        return num_ports;
}

static int edge_startup(struct usb_serial *serial)
{
        struct edgeport_serial *edge_serial;
        int status;
        u16 product_id;

        /* create our private serial structure */
        edge_serial = kzalloc(sizeof(struct edgeport_serial), GFP_KERNEL);
        if (!edge_serial)
                return -ENOMEM;

        mutex_init(&edge_serial->es_lock);
        edge_serial->serial = serial;
        INIT_DELAYED_WORK(&edge_serial->heartbeat_work, edge_heartbeat_work);
        usb_set_serial_data(serial, edge_serial);

        status = download_fw(edge_serial);
        if (status < 0) {
                kfree(edge_serial);
                return status;
        }

        if (status > 0)
                return 1;        /* bind but do not register any ports */

        product_id = le16_to_cpu(
                        edge_serial->serial->dev->descriptor.idProduct);

        /* Currently only the EP/416 models require heartbeat support */
        if (edge_serial->fw_version > FW_HEARTBEAT_VERSION_CUTOFF) {
                if (product_id == ION_DEVICE_ID_TI_EDGEPORT_416 ||
                        product_id == ION_DEVICE_ID_TI_EDGEPORT_416B) {
                        edge_serial->use_heartbeat = true;
                }
        }

        edge_heartbeat_schedule(edge_serial);

        return 0;
}

static void edge_disconnect(struct usb_serial *serial)
{
        struct edgeport_serial *edge_serial = usb_get_serial_data(serial);

        cancel_delayed_work_sync(&edge_serial->heartbeat_work);
}

static void edge_release(struct usb_serial *serial)
{
        struct edgeport_serial *edge_serial = usb_get_serial_data(serial);

        cancel_delayed_work_sync(&edge_serial->heartbeat_work);
        kfree(edge_serial);
}

static int edge_port_probe(struct usb_serial_port *port)
{
        struct edgeport_port *edge_port;
        int ret;

        edge_port = kzalloc(sizeof(*edge_port), GFP_KERNEL);
        if (!edge_port)
                return -ENOMEM;

        spin_lock_init(&edge_port->ep_lock);
        edge_port->port = port;
        edge_port->edge_serial = usb_get_serial_data(port->serial);
        edge_port->bUartMode = default_uart_mode;

        switch (port->port_number) {
        case 0:
                edge_port->uart_base = UMPMEM_BASE_UART1;
                edge_port->dma_address = UMPD_OEDB1_ADDRESS;
                break;
        case 1:
                edge_port->uart_base = UMPMEM_BASE_UART2;
                edge_port->dma_address = UMPD_OEDB2_ADDRESS;
                break;
        default:
                dev_err(&port->dev, "unknown port number\n");
                ret = -ENODEV;
                goto err;
        }

        dev_dbg(&port->dev,
                "%s - port_number = %d, uart_base = %04x, dma_address = %04x\n",
                __func__, port->port_number, edge_port->uart_base,
                edge_port->dma_address);

        usb_set_serial_port_data(port, edge_port);

        ret = edge_create_sysfs_attrs(port);
        if (ret)
                goto err;

        /*
         * The LSR does not tell when the transmitter shift register has
         * emptied so add a one-character drain delay.
         */
        port->port.drain_delay = 1;

        return 0;
err:
        kfree(edge_port);

        return ret;
}

static void edge_port_remove(struct usb_serial_port *port)
{
        struct edgeport_port *edge_port;

        edge_port = usb_get_serial_port_data(port);
        edge_remove_sysfs_attrs(port);
        kfree(edge_port);
}

/* Sysfs Attributes */

static ssize_t uart_mode_show(struct device *dev,
        struct device_attribute *attr, char *buf)
{
        struct usb_serial_port *port = to_usb_serial_port(dev);
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);

        return sprintf(buf, "%d\n", edge_port->bUartMode);
}

static ssize_t uart_mode_store(struct device *dev,
        struct device_attribute *attr, const char *valbuf, size_t count)
{
        struct usb_serial_port *port = to_usb_serial_port(dev);
        struct edgeport_port *edge_port = usb_get_serial_port_data(port);
        unsigned int v = simple_strtoul(valbuf, NULL, 0);

        dev_dbg(dev, "%s: setting uart_mode = %d\n", __func__, v);

        if (v < 256)
                edge_port->bUartMode = v;
        else
                dev_err(dev, "%s - uart_mode %d is invalid\n", __func__, v);

        return count;
}
static DEVICE_ATTR_RW(uart_mode);

static int edge_create_sysfs_attrs(struct usb_serial_port *port)
{
        return device_create_file(&port->dev, &dev_attr_uart_mode);
}

static int edge_remove_sysfs_attrs(struct usb_serial_port *port)
{
        device_remove_file(&port->dev, &dev_attr_uart_mode);
        return 0;
}

#ifdef CONFIG_PM
static int edge_suspend(struct usb_serial *serial, pm_message_t message)
{
        struct edgeport_serial *edge_serial = usb_get_serial_data(serial);

        cancel_delayed_work_sync(&edge_serial->heartbeat_work);

        return 0;
}

static int edge_resume(struct usb_serial *serial)
{
        struct edgeport_serial *edge_serial = usb_get_serial_data(serial);

        edge_heartbeat_schedule(edge_serial);

        return 0;
}
#endif

static struct usb_serial_driver edgeport_1port_device = {
        .driver = {
                .name                = "edgeport_ti_1",
        },
        .description                = "Edgeport TI 1 port adapter",
        .id_table                = edgeport_1port_id_table,
        .num_ports                = 1,
        .num_bulk_out                = 1,
        .open                        = edge_open,
        .close                        = edge_close,
        .throttle                = edge_throttle,
        .unthrottle                = edge_unthrottle,
        .attach                        = edge_startup,
        .calc_num_ports                = edge_calc_num_ports,
        .disconnect                = edge_disconnect,
        .release                = edge_release,
        .port_probe                = edge_port_probe,
        .port_remove                = edge_port_remove,
        .set_termios                = edge_set_termios,
        .tiocmget                = edge_tiocmget,
        .tiocmset                = edge_tiocmset,
        .tiocmiwait                = usb_serial_generic_tiocmiwait,
        .get_icount                = usb_serial_generic_get_icount,
        .write                        = edge_write,
        .write_room                = edge_write_room,
        .chars_in_buffer        = edge_chars_in_buffer,
        .tx_empty                = edge_tx_empty,
        .break_ctl                = edge_break,
        .read_int_callback        = edge_interrupt_callback,
        .read_bulk_callback        = edge_bulk_in_callback,
        .write_bulk_callback        = edge_bulk_out_callback,
#ifdef CONFIG_PM
        .suspend                = edge_suspend,
        .resume                        = edge_resume,
#endif
};

static struct usb_serial_driver edgeport_2port_device = {
        .driver = {
                .name                = "edgeport_ti_2",
        },
        .description                = "Edgeport TI 2 port adapter",
        .id_table                = edgeport_2port_id_table,
        .num_ports                = 2,
        .num_bulk_out                = 1,
        .open                        = edge_open,
        .close                        = edge_close,
        .throttle                = edge_throttle,
        .unthrottle                = edge_unthrottle,
        .attach                        = edge_startup,
        .calc_num_ports                = edge_calc_num_ports,
        .disconnect                = edge_disconnect,
        .release                = edge_release,
        .port_probe                = edge_port_probe,
        .port_remove                = edge_port_remove,
        .set_termios                = edge_set_termios,
        .tiocmget                = edge_tiocmget,
        .tiocmset                = edge_tiocmset,
        .tiocmiwait                = usb_serial_generic_tiocmiwait,
        .get_icount                = usb_serial_generic_get_icount,
        .write                        = edge_write,
        .write_room                = edge_write_room,
        .chars_in_buffer        = edge_chars_in_buffer,
        .tx_empty                = edge_tx_empty,
        .break_ctl                = edge_break,
        .read_int_callback        = edge_interrupt_callback,
        .read_bulk_callback        = edge_bulk_in_callback,
        .write_bulk_callback        = edge_bulk_out_callback,
#ifdef CONFIG_PM
        .suspend                = edge_suspend,
        .resume                        = edge_resume,
#endif
};

static struct usb_serial_driver * const serial_drivers[] = {
        &edgeport_1port_device, &edgeport_2port_device, NULL
};

module_usb_serial_driver(serial_drivers, id_table_combined);

MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
MODULE_LICENSE("GPL");
MODULE_FIRMWARE("edgeport/down3.bin");

module_param(ignore_cpu_rev, bool, 0644);
MODULE_PARM_DESC(ignore_cpu_rev,
                        "Ignore the cpu revision when connecting to a device");

module_param(default_uart_mode, int, 0644);
MODULE_PARM_DESC(default_uart_mode, "Default uart_mode, 0=RS232, ...");



















































    4 

















    4 






    4 



    4 








    4 
    4 














    4 










    4 

    4 



    4 






    4 



    4 

    4 







    4 






    4 




    4 
























    4 




    4 
    4 





    4 








































































































































    4 






























    4 




























    4 
































    4 













































































































































































































































    4 
















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Syntek STK1135 subdriver
 *
 * Copyright (c) 2013 Ondrej Zary
 *
 * Based on Syntekdriver (stk11xx) by Nicolas VIVIEN:
 *   http://syntekdriver.sourceforge.net
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#define MODULE_NAME "stk1135"

#include "gspca.h"
#include "stk1135.h"

MODULE_AUTHOR("Ondrej Zary");
MODULE_DESCRIPTION("Syntek STK1135 USB Camera Driver");
MODULE_LICENSE("GPL");


/* specific webcam descriptor */
struct sd {
        struct gspca_dev gspca_dev;        /* !! must be the first item */

        u8 pkt_seq;
        u8 sensor_page;

        bool flip_status;
        u8 flip_debounce;

        struct v4l2_ctrl *hflip;
        struct v4l2_ctrl *vflip;
};

static const struct v4l2_pix_format stk1135_modes[] = {
        /* default mode (this driver supports variable resolution) */
        {640, 480, V4L2_PIX_FMT_SBGGR8, V4L2_FIELD_NONE,
                .bytesperline = 640,
                .sizeimage = 640 * 480,
                .colorspace = V4L2_COLORSPACE_SRGB},
};

/* -- read a register -- */
static u8 reg_r(struct gspca_dev *gspca_dev, u16 index)
{
        struct usb_device *dev = gspca_dev->dev;
        int ret;

        if (gspca_dev->usb_err < 0)
                return 0;
        ret = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0),
                        0x00,
                        USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
                        0x00,
                        index,
                        gspca_dev->usb_buf, 1,
                        500);

        gspca_dbg(gspca_dev, D_USBI, "reg_r 0x%x=0x%02x\n",
                  index, gspca_dev->usb_buf[0]);
        if (ret < 0) {
                pr_err("reg_r 0x%x err %d\n", index, ret);
                gspca_dev->usb_err = ret;
                return 0;
        }

        return gspca_dev->usb_buf[0];
}

/* -- write a register -- */
static void reg_w(struct gspca_dev *gspca_dev, u16 index, u8 val)
{
        int ret;
        struct usb_device *dev = gspca_dev->dev;

        if (gspca_dev->usb_err < 0)
                return;
        ret = usb_control_msg(dev, usb_sndctrlpipe(dev, 0),
                        0x01,
                        USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
                        val,
                        index,
                        NULL,
                        0,
                        500);
        gspca_dbg(gspca_dev, D_USBO, "reg_w 0x%x:=0x%02x\n", index, val);
        if (ret < 0) {
                pr_err("reg_w 0x%x err %d\n", index, ret);
                gspca_dev->usb_err = ret;
        }
}

static void reg_w_mask(struct gspca_dev *gspca_dev, u16 index, u8 val, u8 mask)
{
        val = (reg_r(gspca_dev, index) & ~mask) | (val & mask);
        reg_w(gspca_dev, index, val);
}

/* this function is called at probe time */
static int sd_config(struct gspca_dev *gspca_dev,
                        const struct usb_device_id *id)
{
        gspca_dev->cam.cam_mode = stk1135_modes;
        gspca_dev->cam.nmodes = ARRAY_SIZE(stk1135_modes);
        return 0;
}

static int stk1135_serial_wait_ready(struct gspca_dev *gspca_dev)
{
        int i = 0;
        u8 val;

        do {
                val = reg_r(gspca_dev, STK1135_REG_SICTL + 1);
                if (i++ > 500) { /* maximum retry count */
                        pr_err("serial bus timeout: status=0x%02x\n", val);
                        return -1;
                }
        /* repeat if BUSY or WRITE/READ not finished */
        } while ((val & 0x10) || !(val & 0x05));

        return 0;
}

static u8 sensor_read_8(struct gspca_dev *gspca_dev, u8 addr)
{
        reg_w(gspca_dev, STK1135_REG_SBUSR, addr);
        /* begin read */
        reg_w(gspca_dev, STK1135_REG_SICTL, 0x20);
        /* wait until finished */
        if (stk1135_serial_wait_ready(gspca_dev)) {
                pr_err("Sensor read failed\n");
                return 0;
        }

        return reg_r(gspca_dev, STK1135_REG_SBUSR + 1);
}

static u16 sensor_read_16(struct gspca_dev *gspca_dev, u8 addr)
{
        return (sensor_read_8(gspca_dev, addr) << 8) |
                sensor_read_8(gspca_dev, 0xf1);
}

static void sensor_write_8(struct gspca_dev *gspca_dev, u8 addr, u8 data)
{
        /* load address and data registers */
        reg_w(gspca_dev, STK1135_REG_SBUSW, addr);
        reg_w(gspca_dev, STK1135_REG_SBUSW + 1, data);
        /* begin write */
        reg_w(gspca_dev, STK1135_REG_SICTL, 0x01);
        /* wait until finished */
        if (stk1135_serial_wait_ready(gspca_dev)) {
                pr_err("Sensor write failed\n");
                return;
        }
}

static void sensor_write_16(struct gspca_dev *gspca_dev, u8 addr, u16 data)
{
        sensor_write_8(gspca_dev, addr, data >> 8);
        sensor_write_8(gspca_dev, 0xf1, data & 0xff);
}

static void sensor_set_page(struct gspca_dev *gspca_dev, u8 page)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (page != sd->sensor_page) {
                sensor_write_16(gspca_dev, 0xf0, page);
                sd->sensor_page = page;
        }
}

static u16 sensor_read(struct gspca_dev *gspca_dev, u16 reg)
{
        sensor_set_page(gspca_dev, reg >> 8);
        return sensor_read_16(gspca_dev, reg & 0xff);
}

static void sensor_write(struct gspca_dev *gspca_dev, u16 reg, u16 val)
{
        sensor_set_page(gspca_dev, reg >> 8);
        sensor_write_16(gspca_dev, reg & 0xff, val);
}

static void sensor_write_mask(struct gspca_dev *gspca_dev,
                        u16 reg, u16 val, u16 mask)
{
        val = (sensor_read(gspca_dev, reg) & ~mask) | (val & mask);
        sensor_write(gspca_dev, reg, val);
}

struct sensor_val {
        u16 reg;
        u16 val;
};

/* configure MT9M112 sensor */
static void stk1135_configure_mt9m112(struct gspca_dev *gspca_dev)
{
        static const struct sensor_val cfg[] = {
                /* restart&reset, chip enable, reserved */
                { 0x00d, 0x000b }, { 0x00d, 0x0008 }, { 0x035, 0x0022 },
                /* mode ctl: AWB on, AE both, clip aper corr, defect corr, AE */
                { 0x106, 0x700e },

                { 0x2dd, 0x18e0 }, /* B-R thresholds, */

                /* AWB */
                { 0x21f, 0x0180 }, /* Cb and Cr limits */
                { 0x220, 0xc814 }, { 0x221, 0x8080 }, /* lum limits, RGB gain */
                { 0x222, 0xa078 }, { 0x223, 0xa078 }, /* R, B limit */
                { 0x224, 0x5f20 }, { 0x228, 0xea02 }, /* mtx adj lim, adv ctl */
                { 0x229, 0x867a }, /* wide gates */

                /* Color correction */
                /* imager gains base, delta, delta signs */
                { 0x25e, 0x594c }, { 0x25f, 0x4d51 }, { 0x260, 0x0002 },
                /* AWB adv ctl 2, gain offs */
                { 0x2ef, 0x0008 }, { 0x2f2, 0x0000 },
                /* base matrix signs, scale K1-5, K6-9 */
                { 0x202, 0x00ee }, { 0x203, 0x3923 }, { 0x204, 0x0724 },
                /* base matrix coef */
                { 0x209, 0x00cd }, { 0x20a, 0x0093 }, { 0x20b, 0x0004 },/*K1-3*/
                { 0x20c, 0x005c }, { 0x20d, 0x00d9 }, { 0x20e, 0x0053 },/*K4-6*/
                { 0x20f, 0x0008 }, { 0x210, 0x0091 }, { 0x211, 0x00cf },/*K7-9*/
                { 0x215, 0x0000 }, /* delta mtx signs */
                /* delta matrix coef */
                { 0x216, 0x0000 }, { 0x217, 0x0000 }, { 0x218, 0x0000 },/*D1-3*/
                { 0x219, 0x0000 }, { 0x21a, 0x0000 }, { 0x21b, 0x0000 },/*D4-6*/
                { 0x21c, 0x0000 }, { 0x21d, 0x0000 }, { 0x21e, 0x0000 },/*D7-9*/
                /* enable & disable manual WB to apply color corr. settings */
                { 0x106, 0xf00e }, { 0x106, 0x700e },

                /* Lens shading correction */
                { 0x180, 0x0007 }, /* control */
                /* vertical knee 0, 2+1, 4+3 */
                { 0x181, 0xde13 }, { 0x182, 0xebe2 }, { 0x183, 0x00f6 }, /* R */
                { 0x184, 0xe114 }, { 0x185, 0xeadd }, { 0x186, 0xfdf6 }, /* G */
                { 0x187, 0xe511 }, { 0x188, 0xede6 }, { 0x189, 0xfbf7 }, /* B */
                /* horizontal knee 0, 2+1, 4+3, 5 */
                { 0x18a, 0xd613 }, { 0x18b, 0xedec }, /* R .. */
                { 0x18c, 0xf9f2 }, { 0x18d, 0x0000 }, /* .. R */
                { 0x18e, 0xd815 }, { 0x18f, 0xe9ea }, /* G .. */
                { 0x190, 0xf9f1 }, { 0x191, 0x0002 }, /* .. G */
                { 0x192, 0xde10 }, { 0x193, 0xefef }, /* B .. */
                { 0x194, 0xfbf4 }, { 0x195, 0x0002 }, /* .. B */
                /* vertical knee 6+5, 8+7 */
                { 0x1b6, 0x0e06 }, { 0x1b7, 0x2713 }, /* R */
                { 0x1b8, 0x1106 }, { 0x1b9, 0x2713 }, /* G */
                { 0x1ba, 0x0c03 }, { 0x1bb, 0x2a0f }, /* B */
                /* horizontal knee 7+6, 9+8, 10 */
                { 0x1bc, 0x1208 }, { 0x1bd, 0x1a16 }, { 0x1be, 0x0022 }, /* R */
                { 0x1bf, 0x150a }, { 0x1c0, 0x1c1a }, { 0x1c1, 0x002d }, /* G */
                { 0x1c2, 0x1109 }, { 0x1c3, 0x1414 }, { 0x1c4, 0x002a }, /* B */
                { 0x106, 0x740e }, /* enable lens shading correction */

                /* Gamma correction - context A */
                { 0x153, 0x0b03 }, { 0x154, 0x4722 }, { 0x155, 0xac82 },
                { 0x156, 0xdac7 }, { 0x157, 0xf5e9 }, { 0x158, 0xff00 },
                /* Gamma correction - context B */
                { 0x1dc, 0x0b03 }, { 0x1dd, 0x4722 }, { 0x1de, 0xac82 },
                { 0x1df, 0xdac7 }, { 0x1e0, 0xf5e9 }, { 0x1e1, 0xff00 },

                /* output format: RGB, invert output pixclock, output bayer */
                { 0x13a, 0x4300 }, { 0x19b, 0x4300 }, /* for context A, B */
                { 0x108, 0x0180 }, /* format control - enable bayer row flip */

                { 0x22f, 0xd100 }, { 0x29c, 0xd100 }, /* AE A, B */

                /* default prg conf, prg ctl - by 0x2d2, prg advance - PA1 */
                { 0x2d2, 0x0000 }, { 0x2cc, 0x0004 }, { 0x2cb, 0x0001 },

                { 0x22e, 0x0c3c }, { 0x267, 0x1010 }, /* AE tgt ctl, gain lim */

                /* PLL */
                { 0x065, 0xa000 }, /* clk ctl - enable PLL (clear bit 14) */
                { 0x066, 0x2003 }, { 0x067, 0x0501 }, /* PLL M=128, N=3, P=1 */
                { 0x065, 0x2000 }, /* disable PLL bypass (clear bit 15) */

                { 0x005, 0x01b8 }, { 0x007, 0x00d8 }, /* horiz blanking B, A */

                /* AE line size, shutter delay limit */
                { 0x239, 0x06c0 }, { 0x23b, 0x040e }, /* for context A */
                { 0x23a, 0x06c0 }, { 0x23c, 0x0564 }, /* for context B */
                /* shutter width basis 60Hz, 50Hz */
                { 0x257, 0x0208 }, { 0x258, 0x0271 }, /* for context A */
                { 0x259, 0x0209 }, { 0x25a, 0x0271 }, /* for context B */

                { 0x25c, 0x120d }, { 0x25d, 0x1712 }, /* flicker 60Hz, 50Hz */
                { 0x264, 0x5e1c }, /* reserved */
                /* flicker, AE gain limits, gain zone limits */
                { 0x25b, 0x0003 }, { 0x236, 0x7810 }, { 0x237, 0x8304 },

                { 0x008, 0x0021 }, /* vert blanking A */
        };
        int i;
        u16 width, height;

        for (i = 0; i < ARRAY_SIZE(cfg); i++)
                sensor_write(gspca_dev, cfg[i].reg, cfg[i].val);

        /* set output size */
        width = gspca_dev->pixfmt.width;
        height = gspca_dev->pixfmt.height;
        if (width <= 640 && height <= 512) { /* context A (half readout speed)*/
                sensor_write(gspca_dev, 0x1a7, width);
                sensor_write(gspca_dev, 0x1aa, height);
                /* set read mode context A */
                sensor_write(gspca_dev, 0x0c8, 0x0000);
                /* set resize, read mode, vblank, hblank context A */
                sensor_write(gspca_dev, 0x2c8, 0x0000);
        } else { /* context B (full readout speed) */
                sensor_write(gspca_dev, 0x1a1, width);
                sensor_write(gspca_dev, 0x1a4, height);
                /* set read mode context B */
                sensor_write(gspca_dev, 0x0c8, 0x0008);
                /* set resize, read mode, vblank, hblank context B */
                sensor_write(gspca_dev, 0x2c8, 0x040b);
        }
}

static void stk1135_configure_clock(struct gspca_dev *gspca_dev)
{
        /* configure SCLKOUT */
        reg_w(gspca_dev, STK1135_REG_TMGEN, 0x12);
        /* set 1 clock per pixel */
        /* and positive edge clocked pulse high when pixel counter = 0 */
        reg_w(gspca_dev, STK1135_REG_TCP1 + 0, 0x41);
        reg_w(gspca_dev, STK1135_REG_TCP1 + 1, 0x00);
        reg_w(gspca_dev, STK1135_REG_TCP1 + 2, 0x00);
        reg_w(gspca_dev, STK1135_REG_TCP1 + 3, 0x00);

        /* enable CLKOUT for sensor */
        reg_w(gspca_dev, STK1135_REG_SENSO + 0, 0x10);
        /* disable STOP clock */
        reg_w(gspca_dev, STK1135_REG_SENSO + 1, 0x00);
        /* set lower 8 bits of PLL feedback divider */
        reg_w(gspca_dev, STK1135_REG_SENSO + 3, 0x07);
        /* set other PLL parameters */
        reg_w(gspca_dev, STK1135_REG_PLLFD, 0x06);
        /* enable timing generator */
        reg_w(gspca_dev, STK1135_REG_TMGEN, 0x80);
        /* enable PLL */
        reg_w(gspca_dev, STK1135_REG_SENSO + 2, 0x04);

        /* set serial interface clock divider (30MHz/0x1f*16+2) = 60240 kHz) */
        reg_w(gspca_dev, STK1135_REG_SICTL + 2, 0x1f);

        /* wait a while for sensor to catch up */
        udelay(1000);
}

static void stk1135_camera_disable(struct gspca_dev *gspca_dev)
{
        /* set capture end Y position to 0 */
        reg_w(gspca_dev, STK1135_REG_CIEPO + 2, 0x00);
        reg_w(gspca_dev, STK1135_REG_CIEPO + 3, 0x00);
        /* disable capture */
        reg_w_mask(gspca_dev, STK1135_REG_SCTRL, 0x00, 0x80);

        /* enable sensor standby and diasble chip enable */
        sensor_write_mask(gspca_dev, 0x00d, 0x0004, 0x000c);

        /* disable PLL */
        reg_w_mask(gspca_dev, STK1135_REG_SENSO + 2, 0x00, 0x01);
        /* disable timing generator */
        reg_w(gspca_dev, STK1135_REG_TMGEN, 0x00);
        /* enable STOP clock */
        reg_w(gspca_dev, STK1135_REG_SENSO + 1, 0x20);
        /* disable CLKOUT for sensor */
        reg_w(gspca_dev, STK1135_REG_SENSO, 0x00);

        /* disable sensor (GPIO5) and enable GPIO0,3,6 (?) - sensor standby? */
        reg_w(gspca_dev, STK1135_REG_GCTRL, 0x49);
}

/* this function is called at probe and resume time */
static int sd_init(struct gspca_dev *gspca_dev)
{
        u16 sensor_id;
        char *sensor_name;
        struct sd *sd = (struct sd *) gspca_dev;

        /* set GPIO3,4,5,6 direction to output */
        reg_w(gspca_dev, STK1135_REG_GCTRL + 2, 0x78);
        /* enable sensor (GPIO5) */
        reg_w(gspca_dev, STK1135_REG_GCTRL, (1 << 5));
        /* disable ROM interface */
        reg_w(gspca_dev, STK1135_REG_GCTRL + 3, 0x80);
        /* enable interrupts from GPIO8 (flip sensor) and GPIO9 (???) */
        reg_w(gspca_dev, STK1135_REG_ICTRL + 1, 0x00);
        reg_w(gspca_dev, STK1135_REG_ICTRL + 3, 0x03);
        /* enable remote wakeup from GPIO9 (???) */
        reg_w(gspca_dev, STK1135_REG_RMCTL + 1, 0x00);
        reg_w(gspca_dev, STK1135_REG_RMCTL + 3, 0x02);

        /* reset serial interface */
        reg_w(gspca_dev, STK1135_REG_SICTL, 0x80);
        reg_w(gspca_dev, STK1135_REG_SICTL, 0x00);
        /* set sensor address */
        reg_w(gspca_dev, STK1135_REG_SICTL + 3, 0xba);
        /* disable alt 2-wire serial interface */
        reg_w(gspca_dev, STK1135_REG_ASIC + 3, 0x00);

        stk1135_configure_clock(gspca_dev);

        /* read sensor ID */
        sd->sensor_page = 0xff;
        sensor_id = sensor_read(gspca_dev, 0x000);

        switch (sensor_id) {
        case 0x148c:
                sensor_name = "MT9M112";
                break;
        default:
                sensor_name = "unknown";
        }
        pr_info("Detected sensor type %s (0x%x)\n", sensor_name, sensor_id);

        stk1135_camera_disable(gspca_dev);

        return gspca_dev->usb_err;
}

/* -- start the camera -- */
static int sd_start(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        u16 width, height;

        /* enable sensor (GPIO5) */
        reg_w(gspca_dev, STK1135_REG_GCTRL, (1 << 5));

        stk1135_configure_clock(gspca_dev);

        /* set capture start position X = 0, Y = 0 */
        reg_w(gspca_dev, STK1135_REG_CISPO + 0, 0x00);
        reg_w(gspca_dev, STK1135_REG_CISPO + 1, 0x00);
        reg_w(gspca_dev, STK1135_REG_CISPO + 2, 0x00);
        reg_w(gspca_dev, STK1135_REG_CISPO + 3, 0x00);

        /* set capture end position */
        width = gspca_dev->pixfmt.width;
        height = gspca_dev->pixfmt.height;
        reg_w(gspca_dev, STK1135_REG_CIEPO + 0, width & 0xff);
        reg_w(gspca_dev, STK1135_REG_CIEPO + 1, width >> 8);
        reg_w(gspca_dev, STK1135_REG_CIEPO + 2, height & 0xff);
        reg_w(gspca_dev, STK1135_REG_CIEPO + 3, height >> 8);

        /* set 8-bit mode */
        reg_w(gspca_dev, STK1135_REG_SCTRL, 0x20);

        stk1135_configure_mt9m112(gspca_dev);

        /* enable capture */
        reg_w_mask(gspca_dev, STK1135_REG_SCTRL, 0x80, 0x80);

        if (gspca_dev->usb_err >= 0)
                gspca_dbg(gspca_dev, D_STREAM, "camera started alt: 0x%02x\n",
                          gspca_dev->alt);

        sd->pkt_seq = 0;

        return gspca_dev->usb_err;
}

static void sd_stopN(struct gspca_dev *gspca_dev)
{
        struct usb_device *dev = gspca_dev->dev;

        usb_set_interface(dev, gspca_dev->iface, 0);

        stk1135_camera_disable(gspca_dev);

        gspca_dbg(gspca_dev, D_STREAM, "camera stopped\n");
}

static void sd_pkt_scan(struct gspca_dev *gspca_dev,
                        u8 *data,                        /* isoc packet */
                        int len)                        /* iso packet length */
{
        struct sd *sd = (struct sd *) gspca_dev;
        int skip = sizeof(struct stk1135_pkt_header);
        bool flip;
        enum gspca_packet_type pkt_type = INTER_PACKET;
        struct stk1135_pkt_header *hdr = (void *)data;
        u8 seq;

        if (len < 4) {
                gspca_dbg(gspca_dev, D_PACK, "received short packet (less than 4 bytes)\n");
                return;
        }

        /* GPIO 8 is flip sensor (1 = normal position, 0 = flipped to back) */
        flip = !(le16_to_cpu(hdr->gpio) & (1 << 8));
        /* it's a switch, needs software debounce */
        if (sd->flip_status != flip)
                sd->flip_debounce++;
        else
                sd->flip_debounce = 0;

        /* check sequence number (not present in new frame packets) */
        if (!(hdr->flags & STK1135_HDR_FRAME_START)) {
                seq = hdr->seq & STK1135_HDR_SEQ_MASK;
                if (seq != sd->pkt_seq) {
                        gspca_dbg(gspca_dev, D_PACK, "received out-of-sequence packet\n");
                        /* resync sequence and discard packet */
                        sd->pkt_seq = seq;
                        gspca_dev->last_packet_type = DISCARD_PACKET;
                        return;
                }
        }
        sd->pkt_seq++;
        if (sd->pkt_seq > STK1135_HDR_SEQ_MASK)
                sd->pkt_seq = 0;

        if (len == sizeof(struct stk1135_pkt_header))
                return;

        if (hdr->flags & STK1135_HDR_FRAME_START) { /* new frame */
                skip = 8;        /* the header is longer */
                gspca_frame_add(gspca_dev, LAST_PACKET, data, 0);
                pkt_type = FIRST_PACKET;
        }
        gspca_frame_add(gspca_dev, pkt_type, data + skip, len - skip);
}

static void sethflip(struct gspca_dev *gspca_dev, s32 val)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (sd->flip_status)
                val = !val;
        sensor_write_mask(gspca_dev, 0x020, val ? 0x0002 : 0x0000 , 0x0002);
}

static void setvflip(struct gspca_dev *gspca_dev, s32 val)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (sd->flip_status)
                val = !val;
        sensor_write_mask(gspca_dev, 0x020, val ? 0x0001 : 0x0000 , 0x0001);
}

static void stk1135_dq_callback(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (sd->flip_debounce > 100) {
                sd->flip_status = !sd->flip_status;
                sethflip(gspca_dev, v4l2_ctrl_g_ctrl(sd->hflip));
                setvflip(gspca_dev, v4l2_ctrl_g_ctrl(sd->vflip));
        }
}

static int sd_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct gspca_dev *gspca_dev =
                container_of(ctrl->handler, struct gspca_dev, ctrl_handler);

        gspca_dev->usb_err = 0;

        if (!gspca_dev->streaming)
                return 0;

        switch (ctrl->id) {
        case V4L2_CID_HFLIP:
                sethflip(gspca_dev, ctrl->val);
                break;
        case V4L2_CID_VFLIP:
                setvflip(gspca_dev, ctrl->val);
                break;
        }

        return gspca_dev->usb_err;
}

static const struct v4l2_ctrl_ops sd_ctrl_ops = {
        .s_ctrl = sd_s_ctrl,
};

static int sd_init_controls(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;
        struct v4l2_ctrl_handler *hdl = &gspca_dev->ctrl_handler;

        gspca_dev->vdev.ctrl_handler = hdl;
        v4l2_ctrl_handler_init(hdl, 2);
        sd->hflip = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                        V4L2_CID_HFLIP, 0, 1, 1, 0);
        sd->vflip = v4l2_ctrl_new_std(hdl, &sd_ctrl_ops,
                        V4L2_CID_VFLIP, 0, 1, 1, 0);

        if (hdl->error) {
                pr_err("Could not initialize controls\n");
                return hdl->error;
        }
        return 0;
}

static void stk1135_try_fmt(struct gspca_dev *gspca_dev, struct v4l2_format *fmt)
{
        fmt->fmt.pix.width = clamp(fmt->fmt.pix.width, 32U, 1280U);
        fmt->fmt.pix.height = clamp(fmt->fmt.pix.height, 32U, 1024U);
        /* round up to even numbers */
        fmt->fmt.pix.width += (fmt->fmt.pix.width & 1);
        fmt->fmt.pix.height += (fmt->fmt.pix.height & 1);

        fmt->fmt.pix.bytesperline = fmt->fmt.pix.width;
        fmt->fmt.pix.sizeimage = fmt->fmt.pix.width * fmt->fmt.pix.height;
}

static int stk1135_enum_framesizes(struct gspca_dev *gspca_dev,
                        struct v4l2_frmsizeenum *fsize)
{
        if (fsize->index != 0 || fsize->pixel_format != V4L2_PIX_FMT_SBGGR8)
                return -EINVAL;

        fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
        fsize->stepwise.min_width = 32;
        fsize->stepwise.min_height = 32;
        fsize->stepwise.max_width = 1280;
        fsize->stepwise.max_height = 1024;
        fsize->stepwise.step_width = 2;
        fsize->stepwise.step_height = 2;

        return 0;
}

/* sub-driver description */
static const struct sd_desc sd_desc = {
        .name = MODULE_NAME,
        .config = sd_config,
        .init = sd_init,
        .init_controls = sd_init_controls,
        .start = sd_start,
        .stopN = sd_stopN,
        .pkt_scan = sd_pkt_scan,
        .dq_callback = stk1135_dq_callback,
        .try_fmt = stk1135_try_fmt,
        .enum_framesizes = stk1135_enum_framesizes,
};

/* -- module initialisation -- */
static const struct usb_device_id device_table[] = {
        {USB_DEVICE(0x174f, 0x6a31)},        /* ASUS laptop, MT9M112 sensor */
        {}
};
MODULE_DEVICE_TABLE(usb, device_table);

/* -- device connect -- */
static int sd_probe(struct usb_interface *intf,
                        const struct usb_device_id *id)
{
        return gspca_dev_probe(intf, id, &sd_desc, sizeof(struct sd),
                                THIS_MODULE);
}

static struct usb_driver sd_driver = {
        .name = MODULE_NAME,
        .id_table = device_table,
        .probe = sd_probe,
        .disconnect = gspca_disconnect,
#ifdef CONFIG_PM
        .suspend = gspca_suspend,
        .resume = gspca_resume,
        .reset_resume = gspca_resume,
#endif
};

module_usb_driver(sd_driver);




































































   76 








    6 
    6 




    4 














   10 

    1 







   19 
   16 










    1 


   18 



   17 
    1 








   16 


   16 
    1 

    1 


    1 




   15 






    1 


    2 
    2 



    3 
   12 
    1 
    1 



   11 






    2 
    1 



   11 
    1 
   11 

   12 



   11 




    9 

    1 



    8 
    8 
    8 


    2 
    2 



   10 





   19 
    9 
    9 
    9 







    2 



















































































































































































































































    5 















    3 

    5 


    5 
    2 


    5 


    5 

    5 










 1157 



 1153 
  247 

   72 

    1 



    1 

   72 

  245 


 1155 






    7 



    7 









    9 
    9 
    9 
    7 


    9 






   10 








    7 









    9 
    2 
    2 
    7 
    4 
    2 
    3 
    1 
    1 
    2 
    2 
    2 


    9 

    9 



    9 
    2 
    7 
    4 
    3 
    1 
    2 
    2 

























   20 

   20 








































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/act_mirred.c        packet mirroring and redirect actions
 *
 * Authors:        Jamal Hadi Salim (2002-4)
 *
 * TODO: Add ingress support (and socket redirect support)
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/gfp.h>
#include <linux/if_arp.h>
#include <net/net_namespace.h>
#include <net/netlink.h>
#include <net/dst.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <linux/tc_act/tc_mirred.h>
#include <net/tc_act/tc_mirred.h>
#include <net/tc_wrapper.h>

static LIST_HEAD(mirred_list);
static DEFINE_SPINLOCK(mirred_list_lock);

#define MIRRED_NEST_LIMIT    4
static DEFINE_PER_CPU(unsigned int, mirred_nest_level);

static bool tcf_mirred_is_act_redirect(int action)
{
        return action == TCA_EGRESS_REDIR || action == TCA_INGRESS_REDIR;
}

static bool tcf_mirred_act_wants_ingress(int action)
{
        switch (action) {
        case TCA_EGRESS_REDIR:
        case TCA_EGRESS_MIRROR:
                return false;
        case TCA_INGRESS_REDIR:
        case TCA_INGRESS_MIRROR:
                return true;
        default:
                BUG();
        }
}

static bool tcf_mirred_can_reinsert(int action)
{
        switch (action) {
        case TC_ACT_SHOT:
        case TC_ACT_STOLEN:
        case TC_ACT_QUEUED:
        case TC_ACT_TRAP:
                return true;
        }
        return false;
}

static struct net_device *tcf_mirred_dev_dereference(struct tcf_mirred *m)
{
        return rcu_dereference_protected(m->tcfm_dev,
                                         lockdep_is_held(&m->tcf_lock));
}

static void tcf_mirred_release(struct tc_action *a)
{
        struct tcf_mirred *m = to_mirred(a);
        struct net_device *dev;

        spin_lock(&mirred_list_lock);
        list_del(&m->tcfm_list);
        spin_unlock(&mirred_list_lock);

        /* last reference to action, no need to lock */
        dev = rcu_dereference_protected(m->tcfm_dev, 1);
        netdev_put(dev, &m->tcfm_dev_tracker);
}

static const struct nla_policy mirred_policy[TCA_MIRRED_MAX + 1] = {
        [TCA_MIRRED_PARMS]        = { .len = sizeof(struct tc_mirred) },
        [TCA_MIRRED_BLOCKID]        = NLA_POLICY_MIN(NLA_U32, 1),
};

static struct tc_action_ops act_mirred_ops;

static void tcf_mirred_replace_dev(struct tcf_mirred *m,
                                   struct net_device *ndev)
{
        struct net_device *odev;

        odev = rcu_replace_pointer(m->tcfm_dev, ndev,
                                   lockdep_is_held(&m->tcf_lock));
        netdev_put(odev, &m->tcfm_dev_tracker);
}

static int tcf_mirred_init(struct net *net, struct nlattr *nla,
                           struct nlattr *est, struct tc_action **a,
                           struct tcf_proto *tp,
                           u32 flags, struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct nlattr *tb[TCA_MIRRED_MAX + 1];
        struct tcf_chain *goto_ch = NULL;
        bool mac_header_xmit = false;
        struct tc_mirred *parm;
        struct tcf_mirred *m;
        bool exists = false;
        int ret, err;
        u32 index;

        if (!nla) {
                NL_SET_ERR_MSG_MOD(extack, "Mirred requires attributes to be passed");
                return -EINVAL;
        }
        ret = nla_parse_nested_deprecated(tb, TCA_MIRRED_MAX, nla,
                                          mirred_policy, extack);
        if (ret < 0)
                return ret;
        if (!tb[TCA_MIRRED_PARMS]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing required mirred parameters");
                return -EINVAL;
        }
        parm = nla_data(tb[TCA_MIRRED_PARMS]);
        index = parm->index;
        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (err < 0)
                return err;
        exists = err;
        if (exists && bind)
                return ACT_P_BOUND;

        if (tb[TCA_MIRRED_BLOCKID] && parm->ifindex) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Cannot specify Block ID and dev simultaneously");
                if (exists)
                        tcf_idr_release(*a, bind);
                else
                        tcf_idr_cleanup(tn, index);

                return -EINVAL;
        }

        switch (parm->eaction) {
        case TCA_EGRESS_MIRROR:
        case TCA_EGRESS_REDIR:
        case TCA_INGRESS_REDIR:
        case TCA_INGRESS_MIRROR:
                break;
        default:
                if (exists)
                        tcf_idr_release(*a, bind);
                else
                        tcf_idr_cleanup(tn, index);
                NL_SET_ERR_MSG_MOD(extack, "Unknown mirred option");
                return -EINVAL;
        }

        if (!exists) {
                if (!parm->ifindex && !tb[TCA_MIRRED_BLOCKID]) {
                        tcf_idr_cleanup(tn, index);
                        NL_SET_ERR_MSG_MOD(extack,
                                           "Must specify device or block");
                        return -EINVAL;
                }
                ret = tcf_idr_create_from_flags(tn, index, est, a,
                                                &act_mirred_ops, bind, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        return ret;
                }
                ret = ACT_P_CREATED;
        } else if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                tcf_idr_release(*a, bind);
                return -EEXIST;
        }

        m = to_mirred(*a);
        if (ret == ACT_P_CREATED)
                INIT_LIST_HEAD(&m->tcfm_list);

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0)
                goto release_idr;

        spin_lock_bh(&m->tcf_lock);

        if (parm->ifindex) {
                struct net_device *ndev;

                ndev = dev_get_by_index(net, parm->ifindex);
                if (!ndev) {
                        spin_unlock_bh(&m->tcf_lock);
                        err = -ENODEV;
                        goto put_chain;
                }
                mac_header_xmit = dev_is_mac_header_xmit(ndev);
                tcf_mirred_replace_dev(m, ndev);
                netdev_tracker_alloc(ndev, &m->tcfm_dev_tracker, GFP_ATOMIC);
                m->tcfm_mac_header_xmit = mac_header_xmit;
                m->tcfm_blockid = 0;
        } else if (tb[TCA_MIRRED_BLOCKID]) {
                tcf_mirred_replace_dev(m, NULL);
                m->tcfm_mac_header_xmit = false;
                m->tcfm_blockid = nla_get_u32(tb[TCA_MIRRED_BLOCKID]);
        }
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
        m->tcfm_eaction = parm->eaction;
        spin_unlock_bh(&m->tcf_lock);
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);

        if (ret == ACT_P_CREATED) {
                spin_lock(&mirred_list_lock);
                list_add(&m->tcfm_list, &mirred_list);
                spin_unlock(&mirred_list_lock);
        }

        return ret;
put_chain:
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
release_idr:
        tcf_idr_release(*a, bind);
        return err;
}

static int
tcf_mirred_forward(bool at_ingress, bool want_ingress, struct sk_buff *skb)
{
        int err;

        if (!want_ingress)
                err = tcf_dev_queue_xmit(skb, dev_queue_xmit);
        else if (!at_ingress)
                err = netif_rx(skb);
        else
                err = netif_receive_skb(skb);

        return err;
}

static int tcf_mirred_to_dev(struct sk_buff *skb, struct tcf_mirred *m,
                             struct net_device *dev,
                             const bool m_mac_header_xmit, int m_eaction,
                             int retval)
{
        struct sk_buff *skb_to_send = skb;
        bool want_ingress;
        bool is_redirect;
        bool expects_nh;
        bool at_ingress;
        bool dont_clone;
        int mac_len;
        bool at_nh;
        int err;

        is_redirect = tcf_mirred_is_act_redirect(m_eaction);
        if (unlikely(!(dev->flags & IFF_UP)) || !netif_carrier_ok(dev)) {
                net_notice_ratelimited("tc mirred to Houston: device %s is down\n",
                                       dev->name);
                goto err_cant_do;
        }

        /* we could easily avoid the clone only if called by ingress and clsact;
         * since we can't easily detect the clsact caller, skip clone only for
         * ingress - that covers the TC S/W datapath.
         */
        at_ingress = skb_at_tc_ingress(skb);
        dont_clone = skb_at_tc_ingress(skb) && is_redirect &&
                tcf_mirred_can_reinsert(retval);
        if (!dont_clone) {
                skb_to_send = skb_clone(skb, GFP_ATOMIC);
                if (!skb_to_send)
                        goto err_cant_do;
        }

        want_ingress = tcf_mirred_act_wants_ingress(m_eaction);

        /* All mirred/redirected skbs should clear previous ct info */
        nf_reset_ct(skb_to_send);
        if (want_ingress && !at_ingress) /* drop dst for egress -> ingress */
                skb_dst_drop(skb_to_send);

        expects_nh = want_ingress || !m_mac_header_xmit;
        at_nh = skb->data == skb_network_header(skb);
        if (at_nh != expects_nh) {
                mac_len = at_ingress ? skb->mac_len :
                          skb_network_offset(skb);
                if (expects_nh) {
                        /* target device/action expect data at nh */
                        skb_pull_rcsum(skb_to_send, mac_len);
                } else {
                        /* target device/action expect data at mac */
                        skb_push_rcsum(skb_to_send, mac_len);
                }
        }

        skb_to_send->skb_iif = skb->dev->ifindex;
        skb_to_send->dev = dev;

        if (is_redirect) {
                if (skb == skb_to_send)
                        retval = TC_ACT_CONSUMED;

                skb_set_redirected(skb_to_send, skb_to_send->tc_at_ingress);

                err = tcf_mirred_forward(at_ingress, want_ingress, skb_to_send);
        } else {
                err = tcf_mirred_forward(at_ingress, want_ingress, skb_to_send);
        }
        if (err)
                tcf_action_inc_overlimit_qstats(&m->common);

        return retval;

err_cant_do:
        if (is_redirect)
                retval = TC_ACT_SHOT;
        tcf_action_inc_overlimit_qstats(&m->common);
        return retval;
}

static int tcf_blockcast_redir(struct sk_buff *skb, struct tcf_mirred *m,
                               struct tcf_block *block, int m_eaction,
                               const u32 exception_ifindex, int retval)
{
        struct net_device *dev_prev = NULL;
        struct net_device *dev = NULL;
        unsigned long index;
        int mirred_eaction;

        mirred_eaction = tcf_mirred_act_wants_ingress(m_eaction) ?
                TCA_INGRESS_MIRROR : TCA_EGRESS_MIRROR;

        xa_for_each(&block->ports, index, dev) {
                if (index == exception_ifindex)
                        continue;

                if (!dev_prev)
                        goto assign_prev;

                tcf_mirred_to_dev(skb, m, dev_prev,
                                  dev_is_mac_header_xmit(dev),
                                  mirred_eaction, retval);
assign_prev:
                dev_prev = dev;
        }

        if (dev_prev)
                return tcf_mirred_to_dev(skb, m, dev_prev,
                                         dev_is_mac_header_xmit(dev_prev),
                                         m_eaction, retval);

        return retval;
}

static int tcf_blockcast_mirror(struct sk_buff *skb, struct tcf_mirred *m,
                                struct tcf_block *block, int m_eaction,
                                const u32 exception_ifindex, int retval)
{
        struct net_device *dev = NULL;
        unsigned long index;

        xa_for_each(&block->ports, index, dev) {
                if (index == exception_ifindex)
                        continue;

                tcf_mirred_to_dev(skb, m, dev,
                                  dev_is_mac_header_xmit(dev),
                                  m_eaction, retval);
        }

        return retval;
}

static int tcf_blockcast(struct sk_buff *skb, struct tcf_mirred *m,
                         const u32 blockid, struct tcf_result *res,
                         int retval)
{
        const u32 exception_ifindex = skb->dev->ifindex;
        struct tcf_block *block;
        bool is_redirect;
        int m_eaction;

        m_eaction = READ_ONCE(m->tcfm_eaction);
        is_redirect = tcf_mirred_is_act_redirect(m_eaction);

        /* we are already under rcu protection, so can call block lookup
         * directly.
         */
        block = tcf_block_lookup(dev_net(skb->dev), blockid);
        if (!block || xa_empty(&block->ports)) {
                tcf_action_inc_overlimit_qstats(&m->common);
                return retval;
        }

        if (is_redirect)
                return tcf_blockcast_redir(skb, m, block, m_eaction,
                                           exception_ifindex, retval);

        /* If it's not redirect, it is mirror */
        return tcf_blockcast_mirror(skb, m, block, m_eaction, exception_ifindex,
                                    retval);
}

TC_INDIRECT_SCOPE int tcf_mirred_act(struct sk_buff *skb,
                                     const struct tc_action *a,
                                     struct tcf_result *res)
{
        struct tcf_mirred *m = to_mirred(a);
        int retval = READ_ONCE(m->tcf_action);
        unsigned int nest_level;
        bool m_mac_header_xmit;
        struct net_device *dev;
        int m_eaction;
        u32 blockid;

        nest_level = __this_cpu_inc_return(mirred_nest_level);
        if (unlikely(nest_level > MIRRED_NEST_LIMIT)) {
                net_warn_ratelimited("Packet exceeded mirred recursion limit on dev %s\n",
                                     netdev_name(skb->dev));
                retval = TC_ACT_SHOT;
                goto dec_nest_level;
        }

        tcf_lastuse_update(&m->tcf_tm);
        tcf_action_update_bstats(&m->common, skb);

        blockid = READ_ONCE(m->tcfm_blockid);
        if (blockid) {
                retval = tcf_blockcast(skb, m, blockid, res, retval);
                goto dec_nest_level;
        }

        dev = rcu_dereference_bh(m->tcfm_dev);
        if (unlikely(!dev)) {
                pr_notice_once("tc mirred: target device is gone\n");
                tcf_action_inc_overlimit_qstats(&m->common);
                goto dec_nest_level;
        }

        m_mac_header_xmit = READ_ONCE(m->tcfm_mac_header_xmit);
        m_eaction = READ_ONCE(m->tcfm_eaction);

        retval = tcf_mirred_to_dev(skb, m, dev, m_mac_header_xmit, m_eaction,
                                   retval);

dec_nest_level:
        __this_cpu_dec(mirred_nest_level);

        return retval;
}

static void tcf_stats_update(struct tc_action *a, u64 bytes, u64 packets,
                             u64 drops, u64 lastuse, bool hw)
{
        struct tcf_mirred *m = to_mirred(a);
        struct tcf_t *tm = &m->tcf_tm;

        tcf_action_update_stats(a, bytes, packets, drops, hw);
        tm->lastuse = max_t(u64, tm->lastuse, lastuse);
}

static int tcf_mirred_dump(struct sk_buff *skb, struct tc_action *a, int bind,
                           int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        struct tcf_mirred *m = to_mirred(a);
        struct tc_mirred opt = {
                .index   = m->tcf_index,
                .refcnt  = refcount_read(&m->tcf_refcnt) - ref,
                .bindcnt = atomic_read(&m->tcf_bindcnt) - bind,
        };
        struct net_device *dev;
        struct tcf_t t;
        u32 blockid;

        spin_lock_bh(&m->tcf_lock);
        opt.action = m->tcf_action;
        opt.eaction = m->tcfm_eaction;
        dev = tcf_mirred_dev_dereference(m);
        if (dev)
                opt.ifindex = dev->ifindex;

        if (nla_put(skb, TCA_MIRRED_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        blockid = m->tcfm_blockid;
        if (blockid && nla_put_u32(skb, TCA_MIRRED_BLOCKID, blockid))
                goto nla_put_failure;

        tcf_tm_dump(&t, &m->tcf_tm);
        if (nla_put_64bit(skb, TCA_MIRRED_TM, sizeof(t), &t, TCA_MIRRED_PAD))
                goto nla_put_failure;
        spin_unlock_bh(&m->tcf_lock);

        return skb->len;

nla_put_failure:
        spin_unlock_bh(&m->tcf_lock);
        nlmsg_trim(skb, b);
        return -1;
}

static int mirred_device_event(struct notifier_block *unused,
                               unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct tcf_mirred *m;

        ASSERT_RTNL();
        if (event == NETDEV_UNREGISTER) {
                spin_lock(&mirred_list_lock);
                list_for_each_entry(m, &mirred_list, tcfm_list) {
                        spin_lock_bh(&m->tcf_lock);
                        if (tcf_mirred_dev_dereference(m) == dev) {
                                netdev_put(dev, &m->tcfm_dev_tracker);
                                /* Note : no rcu grace period necessary, as
                                 * net_device are already rcu protected.
                                 */
                                RCU_INIT_POINTER(m->tcfm_dev, NULL);
                        }
                        spin_unlock_bh(&m->tcf_lock);
                }
                spin_unlock(&mirred_list_lock);
        }

        return NOTIFY_DONE;
}

static struct notifier_block mirred_device_notifier = {
        .notifier_call = mirred_device_event,
};

static void tcf_mirred_dev_put(void *priv)
{
        struct net_device *dev = priv;

        dev_put(dev);
}

static struct net_device *
tcf_mirred_get_dev(const struct tc_action *a,
                   tc_action_priv_destructor *destructor)
{
        struct tcf_mirred *m = to_mirred(a);
        struct net_device *dev;

        rcu_read_lock();
        dev = rcu_dereference(m->tcfm_dev);
        if (dev) {
                dev_hold(dev);
                *destructor = tcf_mirred_dev_put;
        }
        rcu_read_unlock();

        return dev;
}

static size_t tcf_mirred_get_fill_size(const struct tc_action *act)
{
        return nla_total_size(sizeof(struct tc_mirred));
}

static void tcf_offload_mirred_get_dev(struct flow_action_entry *entry,
                                       const struct tc_action *act)
{
        entry->dev = act->ops->get_dev(act, &entry->destructor);
        if (!entry->dev)
                return;
        entry->destructor_priv = entry->dev;
}

static int tcf_mirred_offload_act_setup(struct tc_action *act, void *entry_data,
                                        u32 *index_inc, bool bind,
                                        struct netlink_ext_ack *extack)
{
        if (bind) {
                struct flow_action_entry *entry = entry_data;

                if (is_tcf_mirred_egress_redirect(act)) {
                        entry->id = FLOW_ACTION_REDIRECT;
                        tcf_offload_mirred_get_dev(entry, act);
                } else if (is_tcf_mirred_egress_mirror(act)) {
                        entry->id = FLOW_ACTION_MIRRED;
                        tcf_offload_mirred_get_dev(entry, act);
                } else if (is_tcf_mirred_ingress_redirect(act)) {
                        entry->id = FLOW_ACTION_REDIRECT_INGRESS;
                        tcf_offload_mirred_get_dev(entry, act);
                } else if (is_tcf_mirred_ingress_mirror(act)) {
                        entry->id = FLOW_ACTION_MIRRED_INGRESS;
                        tcf_offload_mirred_get_dev(entry, act);
                } else {
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported mirred offload");
                        return -EOPNOTSUPP;
                }
                *index_inc = 1;
        } else {
                struct flow_offload_action *fl_action = entry_data;

                if (is_tcf_mirred_egress_redirect(act))
                        fl_action->id = FLOW_ACTION_REDIRECT;
                else if (is_tcf_mirred_egress_mirror(act))
                        fl_action->id = FLOW_ACTION_MIRRED;
                else if (is_tcf_mirred_ingress_redirect(act))
                        fl_action->id = FLOW_ACTION_REDIRECT_INGRESS;
                else if (is_tcf_mirred_ingress_mirror(act))
                        fl_action->id = FLOW_ACTION_MIRRED_INGRESS;
                else
                        return -EOPNOTSUPP;
        }

        return 0;
}

static struct tc_action_ops act_mirred_ops = {
        .kind                =        "mirred",
        .id                =        TCA_ID_MIRRED,
        .owner                =        THIS_MODULE,
        .act                =        tcf_mirred_act,
        .stats_update        =        tcf_stats_update,
        .dump                =        tcf_mirred_dump,
        .cleanup        =        tcf_mirred_release,
        .init                =        tcf_mirred_init,
        .get_fill_size        =        tcf_mirred_get_fill_size,
        .offload_act_setup =        tcf_mirred_offload_act_setup,
        .size                =        sizeof(struct tcf_mirred),
        .get_dev        =        tcf_mirred_get_dev,
};
MODULE_ALIAS_NET_ACT("mirred");

static __net_init int mirred_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id);

        return tc_action_net_init(net, tn, &act_mirred_ops);
}

static void __net_exit mirred_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, act_mirred_ops.net_id);
}

static struct pernet_operations mirred_net_ops = {
        .init = mirred_init_net,
        .exit_batch = mirred_exit_net,
        .id   = &act_mirred_ops.net_id,
        .size = sizeof(struct tc_action_net),
};

MODULE_AUTHOR("Jamal Hadi Salim(2002)");
MODULE_DESCRIPTION("Device Mirror/redirect actions");
MODULE_LICENSE("GPL");

static int __init mirred_init_module(void)
{
        int err = register_netdevice_notifier(&mirred_device_notifier);
        if (err)
                return err;

        pr_info("Mirror/redirect action on\n");
        err = tcf_register_action(&act_mirred_ops, &mirred_net_ops);
        if (err)
                unregister_netdevice_notifier(&mirred_device_notifier);

        return err;
}

static void __exit mirred_cleanup_module(void)
{
        tcf_unregister_action(&act_mirred_ops, &mirred_net_ops);
        unregister_netdevice_notifier(&mirred_device_notifier);
}

module_init(mirred_init_module);
module_exit(mirred_cleanup_module);






























    6 




    6 
    1 


    5 



    3 
    2 
    2 
    1 








   45 
   41 

    6 









    2 









    2 
    2 






    2 





    2 
    2 


    2 


    2 
    2 




















    2 











   10 














   10 


















































    8 


    8 

    8 











   10 






    8 





    8 




   10 

   10 


   10 



   10 




   10 
    7 


    1 





   10 















   10 










    8 
    8 






    2 



    2 









   10 


   10 








    7 








    7 

















    7 
    7 



    7 


    7 


    7 




    7 
    7 
    7 






    7 


















































































































   51 






   41 
   41 


   10 

    1 




    9 


    5 
    7 





    7 





    7 
    6 

    2 







    9 
   51 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2019 HUAWEI, Inc.
 *             https://www.huawei.com/
 * Copyright (C) 2024 Alibaba Cloud
 */
#include "compress.h"
#include <linux/lz4.h>

#ifndef LZ4_DISTANCE_MAX        /* history window size */
#define LZ4_DISTANCE_MAX 65535        /* set to maximum value by default */
#endif

#define LZ4_MAX_DISTANCE_PAGES        (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1)
#ifndef LZ4_DECOMPRESS_INPLACE_MARGIN
#define LZ4_DECOMPRESS_INPLACE_MARGIN(srcsize)  (((srcsize) >> 8) + 32)
#endif

struct z_erofs_lz4_decompress_ctx {
        struct z_erofs_decompress_req *rq;
        /* # of encoded, decoded pages */
        unsigned int inpages, outpages;
        /* decoded block total length (used for in-place decompression) */
        unsigned int oend;
};

static int z_erofs_load_lz4_config(struct super_block *sb,
                            struct erofs_super_block *dsb, void *data, int size)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        struct z_erofs_lz4_cfgs *lz4 = data;
        u16 distance;

        if (lz4) {
                if (size < sizeof(struct z_erofs_lz4_cfgs)) {
                        erofs_err(sb, "invalid lz4 cfgs, size=%u", size);
                        return -EINVAL;
                }
                distance = le16_to_cpu(lz4->max_distance);

                sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks);
                if (!sbi->lz4.max_pclusterblks) {
                        sbi->lz4.max_pclusterblks = 1;        /* reserved case */
                } else if (sbi->lz4.max_pclusterblks >
                           erofs_blknr(sb, Z_EROFS_PCLUSTER_MAX_SIZE)) {
                        erofs_err(sb, "too large lz4 pclusterblks %u",
                                  sbi->lz4.max_pclusterblks);
                        return -EINVAL;
                }
        } else {
                distance = le16_to_cpu(dsb->u1.lz4_max_distance);
                sbi->lz4.max_pclusterblks = 1;
        }

        sbi->lz4.max_distance_pages = distance ?
                                        DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
                                        LZ4_MAX_DISTANCE_PAGES;
        return z_erofs_gbuf_growsize(sbi->lz4.max_pclusterblks);
}

/*
 * Fill all gaps with bounce pages if it's a sparse page list. Also check if
 * all physical pages are consecutive, which can be seen for moderate CR.
 */
static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx,
                                        struct page **pagepool)
{
        struct z_erofs_decompress_req *rq = ctx->rq;
        struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
        unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
                                           BITS_PER_LONG)] = { 0 };
        unsigned int lz4_max_distance_pages =
                                EROFS_SB(rq->sb)->lz4.max_distance_pages;
        void *kaddr = NULL;
        unsigned int i, j, top;

        top = 0;
        for (i = j = 0; i < ctx->outpages; ++i, ++j) {
                struct page *const page = rq->out[i];
                struct page *victim;

                if (j >= lz4_max_distance_pages)
                        j = 0;

                /* 'valid' bounced can only be tested after a complete round */
                if (!rq->fillgaps && test_bit(j, bounced)) {
                        DBG_BUGON(i < lz4_max_distance_pages);
                        DBG_BUGON(top >= lz4_max_distance_pages);
                        availables[top++] = rq->out[i - lz4_max_distance_pages];
                }

                if (page) {
                        __clear_bit(j, bounced);
                        if (!PageHighMem(page)) {
                                if (!i) {
                                        kaddr = page_address(page);
                                        continue;
                                }
                                if (kaddr &&
                                    kaddr + PAGE_SIZE == page_address(page)) {
                                        kaddr += PAGE_SIZE;
                                        continue;
                                }
                        }
                        kaddr = NULL;
                        continue;
                }
                kaddr = NULL;
                __set_bit(j, bounced);

                if (top) {
                        victim = availables[--top];
                } else {
                        victim = __erofs_allocpage(pagepool, rq->gfp, true);
                        if (!victim)
                                return -ENOMEM;
                        set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
                }
                rq->out[i] = victim;
        }
        return kaddr ? 1 : 0;
}

static void *z_erofs_lz4_handle_overlap(struct z_erofs_lz4_decompress_ctx *ctx,
                        void *inpage, void *out, unsigned int *inputmargin,
                        int *maptype, bool may_inplace)
{
        struct z_erofs_decompress_req *rq = ctx->rq;
        unsigned int omargin, total, i;
        struct page **in;
        void *src, *tmp;

        if (rq->inplace_io) {
                omargin = PAGE_ALIGN(ctx->oend) - ctx->oend;
                if (rq->partial_decoding || !may_inplace ||
                    omargin < LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize))
                        goto docopy;

                for (i = 0; i < ctx->inpages; ++i)
                        if (rq->out[ctx->outpages - ctx->inpages + i] !=
                            rq->in[i])
                                goto docopy;
                kunmap_local(inpage);
                *maptype = 3;
                return out + ((ctx->outpages - ctx->inpages) << PAGE_SHIFT);
        }

        if (ctx->inpages <= 1) {
                *maptype = 0;
                return inpage;
        }
        kunmap_local(inpage);
        src = erofs_vm_map_ram(rq->in, ctx->inpages);
        if (!src)
                return ERR_PTR(-ENOMEM);
        *maptype = 1;
        return src;

docopy:
        /* Or copy compressed data which can be overlapped to per-CPU buffer */
        in = rq->in;
        src = z_erofs_get_gbuf(ctx->inpages);
        if (!src) {
                DBG_BUGON(1);
                kunmap_local(inpage);
                return ERR_PTR(-EFAULT);
        }

        tmp = src;
        total = rq->inputsize;
        while (total) {
                unsigned int page_copycnt =
                        min_t(unsigned int, total, PAGE_SIZE - *inputmargin);

                if (!inpage)
                        inpage = kmap_local_page(*in);
                memcpy(tmp, inpage + *inputmargin, page_copycnt);
                kunmap_local(inpage);
                inpage = NULL;
                tmp += page_copycnt;
                total -= page_copycnt;
                ++in;
                *inputmargin = 0;
        }
        *maptype = 2;
        return src;
}

/*
 * Get the exact inputsize with zero_padding feature.
 *  - For LZ4, it should work if zero_padding feature is on (5.3+);
 *  - For MicroLZMA, it'd be enabled all the time.
 */
int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf,
                         unsigned int padbufsize)
{
        const char *padend;

        padend = memchr_inv(padbuf, 0, padbufsize);
        if (!padend)
                return -EFSCORRUPTED;
        rq->inputsize -= padend - padbuf;
        rq->pageofs_in += padend - padbuf;
        return 0;
}

static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
                                      u8 *dst)
{
        struct z_erofs_decompress_req *rq = ctx->rq;
        bool support_0padding = false, may_inplace = false;
        unsigned int inputmargin;
        u8 *out, *headpage, *src;
        int ret, maptype;

        DBG_BUGON(*rq->in == NULL);
        headpage = kmap_local_page(*rq->in);

        /* LZ4 decompression inplace is only safe if zero_padding is enabled */
        if (erofs_sb_has_zero_padding(EROFS_SB(rq->sb))) {
                support_0padding = true;
                ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
                                min_t(unsigned int, rq->inputsize,
                                      rq->sb->s_blocksize - rq->pageofs_in));
                if (ret) {
                        kunmap_local(headpage);
                        return ret;
                }
                may_inplace = !((rq->pageofs_in + rq->inputsize) &
                                (rq->sb->s_blocksize - 1));
        }

        inputmargin = rq->pageofs_in;
        src = z_erofs_lz4_handle_overlap(ctx, headpage, dst, &inputmargin,
                                         &maptype, may_inplace);
        if (IS_ERR(src))
                return PTR_ERR(src);

        out = dst + rq->pageofs_out;
        /* legacy format could compress extra data in a pcluster. */
        if (rq->partial_decoding || !support_0padding)
                ret = LZ4_decompress_safe_partial(src + inputmargin, out,
                                rq->inputsize, rq->outputsize, rq->outputsize);
        else
                ret = LZ4_decompress_safe(src + inputmargin, out,
                                          rq->inputsize, rq->outputsize);

        if (ret != rq->outputsize) {
                erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
                          ret, rq->inputsize, inputmargin, rq->outputsize);
                if (ret >= 0)
                        memset(out + ret, 0, rq->outputsize - ret);
                ret = -EFSCORRUPTED;
        } else {
                ret = 0;
        }

        if (maptype == 0) {
                kunmap_local(headpage);
        } else if (maptype == 1) {
                vm_unmap_ram(src, ctx->inpages);
        } else if (maptype == 2) {
                z_erofs_put_gbuf(src);
        } else if (maptype != 3) {
                DBG_BUGON(1);
                return -EFAULT;
        }
        return ret;
}

static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq,
                                  struct page **pagepool)
{
        struct z_erofs_lz4_decompress_ctx ctx;
        unsigned int dst_maptype;
        void *dst;
        int ret;

        ctx.rq = rq;
        ctx.oend = rq->pageofs_out + rq->outputsize;
        ctx.outpages = PAGE_ALIGN(ctx.oend) >> PAGE_SHIFT;
        ctx.inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;

        /* one optimized fast path only for non bigpcluster cases yet */
        if (ctx.inpages == 1 && ctx.outpages == 1 && !rq->inplace_io) {
                DBG_BUGON(!*rq->out);
                dst = kmap_local_page(*rq->out);
                dst_maptype = 0;
                goto dstmap_out;
        }

        /* general decoding path which can be used for all cases */
        ret = z_erofs_lz4_prepare_dstpages(&ctx, pagepool);
        if (ret < 0) {
                return ret;
        } else if (ret > 0) {
                dst = page_address(*rq->out);
                dst_maptype = 1;
        } else {
                dst = erofs_vm_map_ram(rq->out, ctx.outpages);
                if (!dst)
                        return -ENOMEM;
                dst_maptype = 2;
        }

dstmap_out:
        ret = z_erofs_lz4_decompress_mem(&ctx, dst);
        if (!dst_maptype)
                kunmap_local(dst);
        else if (dst_maptype == 2)
                vm_unmap_ram(dst, ctx.outpages);
        return ret;
}

static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq,
                                   struct page **pagepool)
{
        const unsigned int nrpages_in =
                PAGE_ALIGN(rq->pageofs_in + rq->inputsize) >> PAGE_SHIFT;
        const unsigned int nrpages_out =
                PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
        const unsigned int bs = rq->sb->s_blocksize;
        unsigned int cur = 0, ni = 0, no, pi, po, insz, cnt;
        u8 *kin;

        if (rq->outputsize > rq->inputsize)
                return -EOPNOTSUPP;
        if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) {
                cur = bs - (rq->pageofs_out & (bs - 1));
                pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK;
                cur = min(cur, rq->outputsize);
                if (cur && rq->out[0]) {
                        kin = kmap_local_page(rq->in[nrpages_in - 1]);
                        if (rq->out[0] == rq->in[nrpages_in - 1]) {
                                memmove(kin + rq->pageofs_out, kin + pi, cur);
                                flush_dcache_page(rq->out[0]);
                        } else {
                                memcpy_to_page(rq->out[0], rq->pageofs_out,
                                               kin + pi, cur);
                        }
                        kunmap_local(kin);
                }
                rq->outputsize -= cur;
        }

        for (; rq->outputsize; rq->pageofs_in = 0, cur += PAGE_SIZE, ni++) {
                insz = min(PAGE_SIZE - rq->pageofs_in, rq->outputsize);
                rq->outputsize -= insz;
                if (!rq->in[ni])
                        continue;
                kin = kmap_local_page(rq->in[ni]);
                pi = 0;
                do {
                        no = (rq->pageofs_out + cur + pi) >> PAGE_SHIFT;
                        po = (rq->pageofs_out + cur + pi) & ~PAGE_MASK;
                        DBG_BUGON(no >= nrpages_out);
                        cnt = min(insz - pi, PAGE_SIZE - po);
                        if (rq->out[no] == rq->in[ni]) {
                                memmove(kin + po,
                                        kin + rq->pageofs_in + pi, cnt);
                                flush_dcache_page(rq->out[no]);
                        } else if (rq->out[no]) {
                                memcpy_to_page(rq->out[no], po,
                                               kin + rq->pageofs_in + pi, cnt);
                        }
                        pi += cnt;
                } while (pi < insz);
                kunmap_local(kin);
        }
        DBG_BUGON(ni > nrpages_in);
        return 0;
}

int z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, void **dst,
                               void **src, struct page **pgpl)
{
        struct z_erofs_decompress_req *rq = dctx->rq;
        struct super_block *sb = rq->sb;
        struct page **pgo, *tmppage;
        unsigned int j;

        if (!dctx->avail_out) {
                if (++dctx->no >= dctx->outpages || !rq->outputsize) {
                        erofs_err(sb, "insufficient space for decompressed data");
                        return -EFSCORRUPTED;
                }

                if (dctx->kout)
                        kunmap_local(dctx->kout);
                dctx->avail_out = min(rq->outputsize, PAGE_SIZE - rq->pageofs_out);
                rq->outputsize -= dctx->avail_out;
                pgo = &rq->out[dctx->no];
                if (!*pgo && rq->fillgaps) {                /* deduped */
                        *pgo = erofs_allocpage(pgpl, rq->gfp);
                        if (!*pgo) {
                                dctx->kout = NULL;
                                return -ENOMEM;
                        }
                        set_page_private(*pgo, Z_EROFS_SHORTLIVED_PAGE);
                }
                if (*pgo) {
                        dctx->kout = kmap_local_page(*pgo);
                        *dst = dctx->kout + rq->pageofs_out;
                } else {
                        *dst = dctx->kout = NULL;
                }
                rq->pageofs_out = 0;
        }

        if (dctx->inbuf_pos == dctx->inbuf_sz && rq->inputsize) {
                if (++dctx->ni >= dctx->inpages) {
                        erofs_err(sb, "invalid compressed data");
                        return -EFSCORRUPTED;
                }
                if (dctx->kout) /* unlike kmap(), take care of the orders */
                        kunmap_local(dctx->kout);
                kunmap_local(dctx->kin);

                dctx->inbuf_sz = min_t(u32, rq->inputsize, PAGE_SIZE);
                rq->inputsize -= dctx->inbuf_sz;
                dctx->kin = kmap_local_page(rq->in[dctx->ni]);
                *src = dctx->kin;
                dctx->bounced = false;
                if (dctx->kout) {
                        j = (u8 *)*dst - dctx->kout;
                        dctx->kout = kmap_local_page(rq->out[dctx->no]);
                        *dst = dctx->kout + j;
                }
                dctx->inbuf_pos = 0;
        }

        /*
         * Handle overlapping: Use the given bounce buffer if the input data is
         * under processing; Or utilize short-lived pages from the on-stack page
         * pool, where pages are shared among the same request.  Note that only
         * a few inplace I/O pages need to be doubled.
         */
        if (!dctx->bounced && rq->out[dctx->no] == rq->in[dctx->ni]) {
                memcpy(dctx->bounce, *src, dctx->inbuf_sz);
                *src = dctx->bounce;
                dctx->bounced = true;
        }

        for (j = dctx->ni + 1; j < dctx->inpages; ++j) {
                if (rq->out[dctx->no] != rq->in[j])
                        continue;
                tmppage = erofs_allocpage(pgpl, rq->gfp);
                if (!tmppage)
                        return -ENOMEM;
                set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
                copy_highpage(tmppage, rq->in[j]);
                rq->in[j] = tmppage;
        }
        return 0;
}

const struct z_erofs_decompressor *z_erofs_decomp[] = {
        [Z_EROFS_COMPRESSION_SHIFTED] = &(const struct z_erofs_decompressor) {
                .decompress = z_erofs_transform_plain,
                .name = "shifted"
        },
        [Z_EROFS_COMPRESSION_INTERLACED] = &(const struct z_erofs_decompressor) {
                .decompress = z_erofs_transform_plain,
                .name = "interlaced"
        },
        [Z_EROFS_COMPRESSION_LZ4] = &(const struct z_erofs_decompressor) {
                .config = z_erofs_load_lz4_config,
                .decompress = z_erofs_lz4_decompress,
                .init = z_erofs_gbuf_init,
                .exit = z_erofs_gbuf_exit,
                .name = "lz4"
        },
#ifdef CONFIG_EROFS_FS_ZIP_LZMA
        [Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp,
#endif
#ifdef CONFIG_EROFS_FS_ZIP_DEFLATE
        [Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp,
#endif
#ifdef CONFIG_EROFS_FS_ZIP_ZSTD
        [Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp,
#endif
};

int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb)
{
        struct erofs_sb_info *sbi = EROFS_SB(sb);
        struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
        unsigned int algs, alg;
        erofs_off_t offset;
        int size, ret = 0;

        if (!erofs_sb_has_compr_cfgs(sbi)) {
                sbi->available_compr_algs = 1 << Z_EROFS_COMPRESSION_LZ4;
                return z_erofs_load_lz4_config(sb, dsb, NULL, 0);
        }

        sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs);
        if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) {
                erofs_err(sb, "unidentified algorithms %x, please upgrade kernel",
                          sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS);
                return -EOPNOTSUPP;
        }

        erofs_init_metabuf(&buf, sb);
        offset = EROFS_SUPER_OFFSET + sbi->sb_size;
        alg = 0;
        for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) {
                const struct z_erofs_decompressor *dec = z_erofs_decomp[alg];
                void *data;

                if (!(algs & 1))
                        continue;

                data = erofs_read_metadata(sb, &buf, &offset, &size);
                if (IS_ERR(data)) {
                        ret = PTR_ERR(data);
                        break;
                }

                if (alg < Z_EROFS_COMPRESSION_MAX && dec && dec->config) {
                        ret = dec->config(sb, dsb, data, size);
                } else {
                        erofs_err(sb, "algorithm %d isn't enabled on this kernel",
                                  alg);
                        ret = -EOPNOTSUPP;
                }
                kfree(data);
                if (ret)
                        break;
        }
        erofs_put_metabuf(&buf);
        return ret;
}

int __init z_erofs_init_decompressor(void)
{
        int i, err;

        for (i = 0; i < Z_EROFS_COMPRESSION_MAX; ++i) {
                err = z_erofs_decomp[i] ? z_erofs_decomp[i]->init() : 0;
                if (err) {
                        while (i--)
                                if (z_erofs_decomp[i])
                                        z_erofs_decomp[i]->exit();
                        return err;
                }
        }
        return 0;
}

void z_erofs_exit_decompressor(void)
{
        int i;

        for (i = 0; i < Z_EROFS_COMPRESSION_MAX; ++i)
                if (z_erofs_decomp[i])
                        z_erofs_decomp[i]->exit();
}





































































    3 


















    1 

    1 
















































    3 









    3 



    3 







    3 





    3 






    3 
    2 


    2 




    1 




    1 



    1 


    1 





    1 





    1 
































    1 








































































    1 







    1 


    1 


    1 
    1 


    1 



    1 





    1 








    1 


    1 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2003-2013 Jozsef Kadlecsik <kadlec@netfilter.org> */

/* Kernel module implementing an IP set type: the hash:net type */

#include <linux/jhash.h>
#include <linux/module.h>
#include <linux/ip.h>
#include <linux/skbuff.h>
#include <linux/errno.h>
#include <linux/random.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/netlink.h>

#include <linux/netfilter.h>
#include <linux/netfilter/ipset/pfxlen.h>
#include <linux/netfilter/ipset/ip_set.h>
#include <linux/netfilter/ipset/ip_set_hash.h>

#define IPSET_TYPE_REV_MIN        0
/*                                1    Range as input support for IPv4 added */
/*                                2    nomatch flag support added */
/*                                3    Counters support added */
/*                                4    Comments support added */
/*                                5    Forceadd support added */
/*                                6    skbinfo support added */
#define IPSET_TYPE_REV_MAX        7 /* bucketsize, initval support added */

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Jozsef Kadlecsik <kadlec@netfilter.org>");
IP_SET_MODULE_DESC("hash:net", IPSET_TYPE_REV_MIN, IPSET_TYPE_REV_MAX);
MODULE_ALIAS("ip_set_hash:net");

/* Type specific function prefix */
#define HTYPE                hash_net
#define IP_SET_HASH_WITH_NETS

/* IPv4 variant */

/* Member elements  */
struct hash_net4_elem {
        __be32 ip;
        u16 padding0;
        u8 nomatch;
        u8 cidr;
};

/* Common functions */

static bool
hash_net4_data_equal(const struct hash_net4_elem *ip1,
                     const struct hash_net4_elem *ip2,
                     u32 *multi)
{
        return ip1->ip == ip2->ip &&
               ip1->cidr == ip2->cidr;
}

static int
hash_net4_do_data_match(const struct hash_net4_elem *elem)
{
        return elem->nomatch ? -ENOTEMPTY : 1;
}

static void
hash_net4_data_set_flags(struct hash_net4_elem *elem, u32 flags)
{
        elem->nomatch = (flags >> 16) & IPSET_FLAG_NOMATCH;
}

static void
hash_net4_data_reset_flags(struct hash_net4_elem *elem, u8 *flags)
{
        swap(*flags, elem->nomatch);
}

static void
hash_net4_data_netmask(struct hash_net4_elem *elem, u8 cidr)
{
        elem->ip &= ip_set_netmask(cidr);
        elem->cidr = cidr;
}

static bool
hash_net4_data_list(struct sk_buff *skb, const struct hash_net4_elem *data)
{
        u32 flags = data->nomatch ? IPSET_FLAG_NOMATCH : 0;

        if (nla_put_ipaddr4(skb, IPSET_ATTR_IP, data->ip) ||
            nla_put_u8(skb, IPSET_ATTR_CIDR, data->cidr) ||
            (flags &&
             nla_put_net32(skb, IPSET_ATTR_CADT_FLAGS, htonl(flags))))
                goto nla_put_failure;
        return false;

nla_put_failure:
        return true;
}

static void
hash_net4_data_next(struct hash_net4_elem *next,
                    const struct hash_net4_elem *d)
{
        next->ip = d->ip;
}

#define MTYPE                hash_net4
#define HOST_MASK        32
#include "ip_set_hash_gen.h"

static int
hash_net4_kadt(struct ip_set *set, const struct sk_buff *skb,
               const struct xt_action_param *par,
               enum ipset_adt adt, struct ip_set_adt_opt *opt)
{
        const struct hash_net4 *h = set->data;
        ipset_adtfn adtfn = set->variant->adt[adt];
        struct hash_net4_elem e = {
                .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
        };
        struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);

        if (e.cidr == 0)
                return -EINVAL;
        if (adt == IPSET_TEST)
                e.cidr = HOST_MASK;

        ip4addrptr(skb, opt->flags & IPSET_DIM_ONE_SRC, &e.ip);
        e.ip &= ip_set_netmask(e.cidr);

        return adtfn(set, &e, &ext, &opt->ext, opt->cmdflags);
}

static int
hash_net4_uadt(struct ip_set *set, struct nlattr *tb[],
               enum ipset_adt adt, u32 *lineno, u32 flags, bool retried)
{
        struct hash_net4 *h = set->data;
        ipset_adtfn adtfn = set->variant->adt[adt];
        struct hash_net4_elem e = { .cidr = HOST_MASK };
        struct ip_set_ext ext = IP_SET_INIT_UEXT(set);
        u32 ip = 0, ip_to = 0, i = 0;
        int ret;

        if (tb[IPSET_ATTR_LINENO])
                *lineno = nla_get_u32(tb[IPSET_ATTR_LINENO]);

        if (unlikely(!tb[IPSET_ATTR_IP] ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_CADT_FLAGS)))
                return -IPSET_ERR_PROTOCOL;

        ret = ip_set_get_hostipaddr4(tb[IPSET_ATTR_IP], &ip);
        if (ret)
                return ret;

        ret = ip_set_get_extensions(set, tb, &ext);
        if (ret)
                return ret;

        if (tb[IPSET_ATTR_CIDR]) {
                e.cidr = nla_get_u8(tb[IPSET_ATTR_CIDR]);
                if (!e.cidr || e.cidr > HOST_MASK)
                        return -IPSET_ERR_INVALID_CIDR;
        }

        if (tb[IPSET_ATTR_CADT_FLAGS]) {
                u32 cadt_flags = ip_set_get_h32(tb[IPSET_ATTR_CADT_FLAGS]);

                if (cadt_flags & IPSET_FLAG_NOMATCH)
                        flags |= (IPSET_FLAG_NOMATCH << 16);
        }

        if (adt == IPSET_TEST || !tb[IPSET_ATTR_IP_TO]) {
                e.ip = htonl(ip & ip_set_hostmask(e.cidr));
                ret = adtfn(set, &e, &ext, &ext, flags);
                return ip_set_enomatch(ret, flags, adt, set) ? -ret :
                       ip_set_eexist(ret, flags) ? 0 : ret;
        }

        ip_to = ip;
        if (tb[IPSET_ATTR_IP_TO]) {
                ret = ip_set_get_hostipaddr4(tb[IPSET_ATTR_IP_TO], &ip_to);
                if (ret)
                        return ret;
                if (ip_to < ip)
                        swap(ip, ip_to);
                if (ip + UINT_MAX == ip_to)
                        return -IPSET_ERR_HASH_RANGE;
        }

        if (retried)
                ip = ntohl(h->next.ip);
        do {
                i++;
                e.ip = htonl(ip);
                if (i > IPSET_MAX_RANGE) {
                        hash_net4_data_next(&h->next, &e);
                        return -ERANGE;
                }
                ip = ip_set_range_to_cidr(ip, ip_to, &e.cidr);
                ret = adtfn(set, &e, &ext, &ext, flags);
                if (ret && !ip_set_eexist(ret, flags))
                        return ret;

                ret = 0;
        } while (ip++ < ip_to);
        return ret;
}

/* IPv6 variant */

struct hash_net6_elem {
        union nf_inet_addr ip;
        u16 padding0;
        u8 nomatch;
        u8 cidr;
};

/* Common functions */

static bool
hash_net6_data_equal(const struct hash_net6_elem *ip1,
                     const struct hash_net6_elem *ip2,
                     u32 *multi)
{
        return ipv6_addr_equal(&ip1->ip.in6, &ip2->ip.in6) &&
               ip1->cidr == ip2->cidr;
}

static int
hash_net6_do_data_match(const struct hash_net6_elem *elem)
{
        return elem->nomatch ? -ENOTEMPTY : 1;
}

static void
hash_net6_data_set_flags(struct hash_net6_elem *elem, u32 flags)
{
        elem->nomatch = (flags >> 16) & IPSET_FLAG_NOMATCH;
}

static void
hash_net6_data_reset_flags(struct hash_net6_elem *elem, u8 *flags)
{
        swap(*flags, elem->nomatch);
}

static void
hash_net6_data_netmask(struct hash_net6_elem *elem, u8 cidr)
{
        ip6_netmask(&elem->ip, cidr);
        elem->cidr = cidr;
}

static bool
hash_net6_data_list(struct sk_buff *skb, const struct hash_net6_elem *data)
{
        u32 flags = data->nomatch ? IPSET_FLAG_NOMATCH : 0;

        if (nla_put_ipaddr6(skb, IPSET_ATTR_IP, &data->ip.in6) ||
            nla_put_u8(skb, IPSET_ATTR_CIDR, data->cidr) ||
            (flags &&
             nla_put_net32(skb, IPSET_ATTR_CADT_FLAGS, htonl(flags))))
                goto nla_put_failure;
        return false;

nla_put_failure:
        return true;
}

static void
hash_net6_data_next(struct hash_net6_elem *next,
                    const struct hash_net6_elem *d)
{
}

#undef MTYPE
#undef HOST_MASK

#define MTYPE                hash_net6
#define HOST_MASK        128
#define IP_SET_EMIT_CREATE
#include "ip_set_hash_gen.h"

static int
hash_net6_kadt(struct ip_set *set, const struct sk_buff *skb,
               const struct xt_action_param *par,
               enum ipset_adt adt, struct ip_set_adt_opt *opt)
{
        const struct hash_net6 *h = set->data;
        ipset_adtfn adtfn = set->variant->adt[adt];
        struct hash_net6_elem e = {
                .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
        };
        struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);

        if (e.cidr == 0)
                return -EINVAL;
        if (adt == IPSET_TEST)
                e.cidr = HOST_MASK;

        ip6addrptr(skb, opt->flags & IPSET_DIM_ONE_SRC, &e.ip.in6);
        ip6_netmask(&e.ip, e.cidr);

        return adtfn(set, &e, &ext, &opt->ext, opt->cmdflags);
}

static int
hash_net6_uadt(struct ip_set *set, struct nlattr *tb[],
               enum ipset_adt adt, u32 *lineno, u32 flags, bool retried)
{
        ipset_adtfn adtfn = set->variant->adt[adt];
        struct hash_net6_elem e = { .cidr = HOST_MASK };
        struct ip_set_ext ext = IP_SET_INIT_UEXT(set);
        int ret;

        if (tb[IPSET_ATTR_LINENO])
                *lineno = nla_get_u32(tb[IPSET_ATTR_LINENO]);

        if (unlikely(!tb[IPSET_ATTR_IP] ||
                     !ip_set_optattr_netorder(tb, IPSET_ATTR_CADT_FLAGS)))
                return -IPSET_ERR_PROTOCOL;
        if (unlikely(tb[IPSET_ATTR_IP_TO]))
                return -IPSET_ERR_HASH_RANGE_UNSUPPORTED;

        ret = ip_set_get_ipaddr6(tb[IPSET_ATTR_IP], &e.ip);
        if (ret)
                return ret;

        ret = ip_set_get_extensions(set, tb, &ext);
        if (ret)
                return ret;

        if (tb[IPSET_ATTR_CIDR]) {
                e.cidr = nla_get_u8(tb[IPSET_ATTR_CIDR]);
                if (!e.cidr || e.cidr > HOST_MASK)
                        return -IPSET_ERR_INVALID_CIDR;
        }

        ip6_netmask(&e.ip, e.cidr);

        if (tb[IPSET_ATTR_CADT_FLAGS]) {
                u32 cadt_flags = ip_set_get_h32(tb[IPSET_ATTR_CADT_FLAGS]);

                if (cadt_flags & IPSET_FLAG_NOMATCH)
                        flags |= (IPSET_FLAG_NOMATCH << 16);
        }

        ret = adtfn(set, &e, &ext, &ext, flags);

        return ip_set_enomatch(ret, flags, adt, set) ? -ret :
               ip_set_eexist(ret, flags) ? 0 : ret;
}

static struct ip_set_type hash_net_type __read_mostly = {
        .name                = "hash:net",
        .protocol        = IPSET_PROTOCOL,
        .features        = IPSET_TYPE_IP | IPSET_TYPE_NOMATCH,
        .dimension        = IPSET_DIM_ONE,
        .family                = NFPROTO_UNSPEC,
        .revision_min        = IPSET_TYPE_REV_MIN,
        .revision_max        = IPSET_TYPE_REV_MAX,
        .create_flags[IPSET_TYPE_REV_MAX] = IPSET_CREATE_FLAG_BUCKETSIZE,
        .create                = hash_net_create,
        .create_policy        = {
                [IPSET_ATTR_HASHSIZE]        = { .type = NLA_U32 },
                [IPSET_ATTR_MAXELEM]        = { .type = NLA_U32 },
                [IPSET_ATTR_INITVAL]        = { .type = NLA_U32 },
                [IPSET_ATTR_BUCKETSIZE]        = { .type = NLA_U8 },
                [IPSET_ATTR_RESIZE]        = { .type = NLA_U8  },
                [IPSET_ATTR_TIMEOUT]        = { .type = NLA_U32 },
                [IPSET_ATTR_CADT_FLAGS]        = { .type = NLA_U32 },
        },
        .adt_policy        = {
                [IPSET_ATTR_IP]                = { .type = NLA_NESTED },
                [IPSET_ATTR_IP_TO]        = { .type = NLA_NESTED },
                [IPSET_ATTR_CIDR]        = { .type = NLA_U8 },
                [IPSET_ATTR_TIMEOUT]        = { .type = NLA_U32 },
                [IPSET_ATTR_LINENO]        = { .type = NLA_U32 },
                [IPSET_ATTR_CADT_FLAGS]        = { .type = NLA_U32 },
                [IPSET_ATTR_BYTES]        = { .type = NLA_U64 },
                [IPSET_ATTR_PACKETS]        = { .type = NLA_U64 },
                [IPSET_ATTR_COMMENT]        = { .type = NLA_NUL_STRING,
                                            .len  = IPSET_MAX_COMMENT_SIZE },
                [IPSET_ATTR_SKBMARK]        = { .type = NLA_U64 },
                [IPSET_ATTR_SKBPRIO]        = { .type = NLA_U32 },
                [IPSET_ATTR_SKBQUEUE]        = { .type = NLA_U16 },
        },
        .me                = THIS_MODULE,
};

static int __init
hash_net_init(void)
{
        return ip_set_type_register(&hash_net_type);
}

static void __exit
hash_net_fini(void)
{
        rcu_barrier();
        ip_set_type_unregister(&hash_net_type);
}

module_init(hash_net_init);
module_exit(hash_net_fini);


























































  216 


  215 



  213 
   11 
  215 
   12 





   12 


   12 














   12 
















  214 
  213 


  217 





    5 

   44 
    5 


















  205 




  207 





  207 



  208 
  206 
  206 
  207 





  208 






































































  214 


  212 
   13 

    6 




  207 









  208 
  204 
  201 



  202 

  204 
  157 




  205 





   44 



  206 



  206 
















  203 
  208 

  206 
    6 
  206 







  208 




  208 
  208 
  208 


  213 

















































  190 





  182 
  182 
  182 


  182 


  183 




  180 


    1 




  188 

  190 











   40 























  207 




  206 



  204 

  206 






  110 


  110 



  205 


  205 
    1 



  205 





  203 
























  187 










  188 




  188 
   20 



   20 



   20 









  189 



   20 


  189 
  187 
  189 































   28 




















   28 




   28 
   28 
   28 












   27 
   28 


   28 






   28 

   28 











   26 

  828 


  478 




   20 




   26 













  280 



  281 
  275 

  278 
  280 
  279 
  281 












  279 



  279 














   67 



  229 
  261 



    4 



   66 







    4 


    2 
    4 










  256 















   14 




   14 


   14 




    9 
    9 


    5 







    5 

















  202 












  202 






  197 
  201 


  199 




  198 


  199 


  199 






  162 




   97 
   11 




  122 




   68 



  123 





  123 
    5 









  118 

    5 







  118 
  118 
   87 













  109 



  109 


  109 
   89 
   98 



  109 







  108 

   18 




   90 





  114 

  197 
  163 

  199 

  201 





   14 











   14 

   14 
    9 






































  201 
























































    7 
    8 













    7 





    6 




    5 




    4 
    4 


    5 


    7 
    8 
























    8 

    8 

    7 


    8 
    8 
    8 




    8 
















   31 
   31 

   31 


   31 

   31 












   56 





   57 

   56 









   57 

















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
// SPDX-License-Identifier: GPL-2.0-or-later
/* Basic authentication token and access key management
 *
 * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/export.h>
#include <linux/init.h>
#include <linux/poison.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/security.h>
#include <linux/workqueue.h>
#include <linux/random.h>
#include <linux/err.h>
#include "internal.h"

struct kmem_cache *key_jar;
struct rb_root                key_serial_tree; /* tree of keys indexed by serial */
DEFINE_SPINLOCK(key_serial_lock);

struct rb_root        key_user_tree; /* tree of quota records indexed by UID */
DEFINE_SPINLOCK(key_user_lock);

unsigned int key_quota_root_maxkeys = 1000000;        /* root's key count quota */
unsigned int key_quota_root_maxbytes = 25000000; /* root's key space quota */
unsigned int key_quota_maxkeys = 200;                /* general key count quota */
unsigned int key_quota_maxbytes = 20000;        /* general key space quota */

static LIST_HEAD(key_types_list);
static DECLARE_RWSEM(key_types_sem);

/* We serialise key instantiation and link */
DEFINE_MUTEX(key_construction_mutex);

#ifdef KEY_DEBUGGING
void __key_check(const struct key *key)
{
        printk("__key_check: key %p {%08x} should be {%08x}\n",
               key, key->magic, KEY_DEBUG_MAGIC);
        BUG();
}
#endif

/*
 * Get the key quota record for a user, allocating a new record if one doesn't
 * already exist.
 */
struct key_user *key_user_lookup(kuid_t uid)
{
        struct key_user *candidate = NULL, *user;
        struct rb_node *parent, **p;

try_again:
        parent = NULL;
        p = &key_user_tree.rb_node;
        spin_lock(&key_user_lock);

        /* search the tree for a user record with a matching UID */
        while (*p) {
                parent = *p;
                user = rb_entry(parent, struct key_user, node);

                if (uid_lt(uid, user->uid))
                        p = &(*p)->rb_left;
                else if (uid_gt(uid, user->uid))
                        p = &(*p)->rb_right;
                else
                        goto found;
        }

        /* if we get here, we failed to find a match in the tree */
        if (!candidate) {
                /* allocate a candidate user record if we don't already have
                 * one */
                spin_unlock(&key_user_lock);

                user = NULL;
                candidate = kmalloc(sizeof(struct key_user), GFP_KERNEL);
                if (unlikely(!candidate))
                        goto out;

                /* the allocation may have scheduled, so we need to repeat the
                 * search lest someone else added the record whilst we were
                 * asleep */
                goto try_again;
        }

        /* if we get here, then the user record still hadn't appeared on the
         * second pass - so we use the candidate record */
        refcount_set(&candidate->usage, 1);
        atomic_set(&candidate->nkeys, 0);
        atomic_set(&candidate->nikeys, 0);
        candidate->uid = uid;
        candidate->qnkeys = 0;
        candidate->qnbytes = 0;
        spin_lock_init(&candidate->lock);
        mutex_init(&candidate->cons_lock);

        rb_link_node(&candidate->node, parent, p);
        rb_insert_color(&candidate->node, &key_user_tree);
        spin_unlock(&key_user_lock);
        user = candidate;
        goto out;

        /* okay - we found a user record for this UID */
found:
        refcount_inc(&user->usage);
        spin_unlock(&key_user_lock);
        kfree(candidate);
out:
        return user;
}

/*
 * Dispose of a user structure
 */
void key_user_put(struct key_user *user)
{
        if (refcount_dec_and_lock(&user->usage, &key_user_lock)) {
                rb_erase(&user->node, &key_user_tree);
                spin_unlock(&key_user_lock);

                kfree(user);
        }
}

/*
 * Allocate a serial number for a key.  These are assigned randomly to avoid
 * security issues through covert channel problems.
 */
static inline void key_alloc_serial(struct key *key)
{
        struct rb_node *parent, **p;
        struct key *xkey;

        /* propose a random serial number and look for a hole for it in the
         * serial number tree */
        do {
                get_random_bytes(&key->serial, sizeof(key->serial));

                key->serial >>= 1; /* negative numbers are not permitted */
        } while (key->serial < 3);

        spin_lock(&key_serial_lock);

attempt_insertion:
        parent = NULL;
        p = &key_serial_tree.rb_node;

        while (*p) {
                parent = *p;
                xkey = rb_entry(parent, struct key, serial_node);

                if (key->serial < xkey->serial)
                        p = &(*p)->rb_left;
                else if (key->serial > xkey->serial)
                        p = &(*p)->rb_right;
                else
                        goto serial_exists;
        }

        /* we've found a suitable hole - arrange for this key to occupy it */
        rb_link_node(&key->serial_node, parent, p);
        rb_insert_color(&key->serial_node, &key_serial_tree);

        spin_unlock(&key_serial_lock);
        return;

        /* we found a key with the proposed serial number - walk the tree from
         * that point looking for the next unused serial number */
serial_exists:
        for (;;) {
                key->serial++;
                if (key->serial < 3) {
                        key->serial = 3;
                        goto attempt_insertion;
                }

                parent = rb_next(parent);
                if (!parent)
                        goto attempt_insertion;

                xkey = rb_entry(parent, struct key, serial_node);
                if (key->serial < xkey->serial)
                        goto attempt_insertion;
        }
}

/**
 * key_alloc - Allocate a key of the specified type.
 * @type: The type of key to allocate.
 * @desc: The key description to allow the key to be searched out.
 * @uid: The owner of the new key.
 * @gid: The group ID for the new key's group permissions.
 * @cred: The credentials specifying UID namespace.
 * @perm: The permissions mask of the new key.
 * @flags: Flags specifying quota properties.
 * @restrict_link: Optional link restriction for new keyrings.
 *
 * Allocate a key of the specified type with the attributes given.  The key is
 * returned in an uninstantiated state and the caller needs to instantiate the
 * key before returning.
 *
 * The restrict_link structure (if not NULL) will be freed when the
 * keyring is destroyed, so it must be dynamically allocated.
 *
 * The user's key count quota is updated to reflect the creation of the key and
 * the user's key data quota has the default for the key type reserved.  The
 * instantiation function should amend this as necessary.  If insufficient
 * quota is available, -EDQUOT will be returned.
 *
 * The LSM security modules can prevent a key being created, in which case
 * -EACCES will be returned.
 *
 * Returns a pointer to the new key if successful and an error code otherwise.
 *
 * Note that the caller needs to ensure the key type isn't uninstantiated.
 * Internally this can be done by locking key_types_sem.  Externally, this can
 * be done by either never unregistering the key type, or making sure
 * key_alloc() calls don't race with module unloading.
 */
struct key *key_alloc(struct key_type *type, const char *desc,
                      kuid_t uid, kgid_t gid, const struct cred *cred,
                      key_perm_t perm, unsigned long flags,
                      struct key_restriction *restrict_link)
{
        struct key_user *user = NULL;
        struct key *key;
        size_t desclen, quotalen;
        int ret;
        unsigned long irqflags;

        key = ERR_PTR(-EINVAL);
        if (!desc || !*desc)
                goto error;

        if (type->vet_description) {
                ret = type->vet_description(desc);
                if (ret < 0) {
                        key = ERR_PTR(ret);
                        goto error;
                }
        }

        desclen = strlen(desc);
        quotalen = desclen + 1 + type->def_datalen;

        /* get hold of the key tracking for this user */
        user = key_user_lookup(uid);
        if (!user)
                goto no_memory_1;

        /* check that the user's quota permits allocation of another key and
         * its description */
        if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) {
                unsigned maxkeys = uid_eq(uid, GLOBAL_ROOT_UID) ?
                        key_quota_root_maxkeys : key_quota_maxkeys;
                unsigned maxbytes = uid_eq(uid, GLOBAL_ROOT_UID) ?
                        key_quota_root_maxbytes : key_quota_maxbytes;

                spin_lock_irqsave(&user->lock, irqflags);
                if (!(flags & KEY_ALLOC_QUOTA_OVERRUN)) {
                        if (user->qnkeys + 1 > maxkeys ||
                            user->qnbytes + quotalen > maxbytes ||
                            user->qnbytes + quotalen < user->qnbytes)
                                goto no_quota;
                }

                user->qnkeys++;
                user->qnbytes += quotalen;
                spin_unlock_irqrestore(&user->lock, irqflags);
        }

        /* allocate and initialise the key and its description */
        key = kmem_cache_zalloc(key_jar, GFP_KERNEL);
        if (!key)
                goto no_memory_2;

        key->index_key.desc_len = desclen;
        key->index_key.description = kmemdup(desc, desclen + 1, GFP_KERNEL);
        if (!key->index_key.description)
                goto no_memory_3;
        key->index_key.type = type;
        key_set_index_key(&key->index_key);

        refcount_set(&key->usage, 1);
        init_rwsem(&key->sem);
        lockdep_set_class(&key->sem, &type->lock_class);
        key->user = user;
        key->quotalen = quotalen;
        key->datalen = type->def_datalen;
        key->uid = uid;
        key->gid = gid;
        key->perm = perm;
        key->expiry = TIME64_MAX;
        key->restrict_link = restrict_link;
        key->last_used_at = ktime_get_real_seconds();

        if (!(flags & KEY_ALLOC_NOT_IN_QUOTA))
                key->flags |= 1 << KEY_FLAG_IN_QUOTA;
        if (flags & KEY_ALLOC_BUILT_IN)
                key->flags |= 1 << KEY_FLAG_BUILTIN;
        if (flags & KEY_ALLOC_UID_KEYRING)
                key->flags |= 1 << KEY_FLAG_UID_KEYRING;
        if (flags & KEY_ALLOC_SET_KEEP)
                key->flags |= 1 << KEY_FLAG_KEEP;

#ifdef KEY_DEBUGGING
        key->magic = KEY_DEBUG_MAGIC;
#endif

        /* let the security module know about the key */
        ret = security_key_alloc(key, cred, flags);
        if (ret < 0)
                goto security_error;

        /* publish the key by giving it a serial number */
        refcount_inc(&key->domain_tag->usage);
        atomic_inc(&user->nkeys);
        key_alloc_serial(key);

error:
        return key;

security_error:
        kfree(key->description);
        kmem_cache_free(key_jar, key);
        if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) {
                spin_lock_irqsave(&user->lock, irqflags);
                user->qnkeys--;
                user->qnbytes -= quotalen;
                spin_unlock_irqrestore(&user->lock, irqflags);
        }
        key_user_put(user);
        key = ERR_PTR(ret);
        goto error;

no_memory_3:
        kmem_cache_free(key_jar, key);
no_memory_2:
        if (!(flags & KEY_ALLOC_NOT_IN_QUOTA)) {
                spin_lock_irqsave(&user->lock, irqflags);
                user->qnkeys--;
                user->qnbytes -= quotalen;
                spin_unlock_irqrestore(&user->lock, irqflags);
        }
        key_user_put(user);
no_memory_1:
        key = ERR_PTR(-ENOMEM);
        goto error;

no_quota:
        spin_unlock_irqrestore(&user->lock, irqflags);
        key_user_put(user);
        key = ERR_PTR(-EDQUOT);
        goto error;
}
EXPORT_SYMBOL(key_alloc);

/**
 * key_payload_reserve - Adjust data quota reservation for the key's payload
 * @key: The key to make the reservation for.
 * @datalen: The amount of data payload the caller now wants.
 *
 * Adjust the amount of the owning user's key data quota that a key reserves.
 * If the amount is increased, then -EDQUOT may be returned if there isn't
 * enough free quota available.
 *
 * If successful, 0 is returned.
 */
int key_payload_reserve(struct key *key, size_t datalen)
{
        int delta = (int)datalen - key->datalen;
        int ret = 0;

        key_check(key);

        /* contemplate the quota adjustment */
        if (delta != 0 && test_bit(KEY_FLAG_IN_QUOTA, &key->flags)) {
                unsigned maxbytes = uid_eq(key->user->uid, GLOBAL_ROOT_UID) ?
                        key_quota_root_maxbytes : key_quota_maxbytes;
                unsigned long flags;

                spin_lock_irqsave(&key->user->lock, flags);

                if (delta > 0 &&
                    (key->user->qnbytes + delta > maxbytes ||
                     key->user->qnbytes + delta < key->user->qnbytes)) {
                        ret = -EDQUOT;
                }
                else {
                        key->user->qnbytes += delta;
                        key->quotalen += delta;
                }
                spin_unlock_irqrestore(&key->user->lock, flags);
        }

        /* change the recorded data length if that didn't generate an error */
        if (ret == 0)
                key->datalen = datalen;

        return ret;
}
EXPORT_SYMBOL(key_payload_reserve);

/*
 * Change the key state to being instantiated.
 */
static void mark_key_instantiated(struct key *key, int reject_error)
{
        /* Commit the payload before setting the state; barrier versus
         * key_read_state().
         */
        smp_store_release(&key->state,
                          (reject_error < 0) ? reject_error : KEY_IS_POSITIVE);
}

/*
 * Instantiate a key and link it into the target keyring atomically.  Must be
 * called with the target keyring's semaphore writelocked.  The target key's
 * semaphore need not be locked as instantiation is serialised by
 * key_construction_mutex.
 */
static int __key_instantiate_and_link(struct key *key,
                                      struct key_preparsed_payload *prep,
                                      struct key *keyring,
                                      struct key *authkey,
                                      struct assoc_array_edit **_edit)
{
        int ret, awaken;

        key_check(key);
        key_check(keyring);

        awaken = 0;
        ret = -EBUSY;

        mutex_lock(&key_construction_mutex);

        /* can't instantiate twice */
        if (key->state == KEY_IS_UNINSTANTIATED) {
                /* instantiate the key */
                ret = key->type->instantiate(key, prep);

                if (ret == 0) {
                        /* mark the key as being instantiated */
                        atomic_inc(&key->user->nikeys);
                        mark_key_instantiated(key, 0);
                        notify_key(key, NOTIFY_KEY_INSTANTIATED, 0);

                        if (test_and_clear_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags))
                                awaken = 1;

                        /* and link it into the destination keyring */
                        if (keyring) {
                                if (test_bit(KEY_FLAG_KEEP, &keyring->flags))
                                        set_bit(KEY_FLAG_KEEP, &key->flags);

                                __key_link(keyring, key, _edit);
                        }

                        /* disable the authorisation key */
                        if (authkey)
                                key_invalidate(authkey);

                        if (prep->expiry != TIME64_MAX)
                                key_set_expiry(key, prep->expiry);
                }
        }

        mutex_unlock(&key_construction_mutex);

        /* wake up anyone waiting for a key to be constructed */
        if (awaken)
                wake_up_bit(&key->flags, KEY_FLAG_USER_CONSTRUCT);

        return ret;
}

/**
 * key_instantiate_and_link - Instantiate a key and link it into the keyring.
 * @key: The key to instantiate.
 * @data: The data to use to instantiate the keyring.
 * @datalen: The length of @data.
 * @keyring: Keyring to create a link in on success (or NULL).
 * @authkey: The authorisation token permitting instantiation.
 *
 * Instantiate a key that's in the uninstantiated state using the provided data
 * and, if successful, link it in to the destination keyring if one is
 * supplied.
 *
 * If successful, 0 is returned, the authorisation token is revoked and anyone
 * waiting for the key is woken up.  If the key was already instantiated,
 * -EBUSY will be returned.
 */
int key_instantiate_and_link(struct key *key,
                             const void *data,
                             size_t datalen,
                             struct key *keyring,
                             struct key *authkey)
{
        struct key_preparsed_payload prep;
        struct assoc_array_edit *edit = NULL;
        int ret;

        memset(&prep, 0, sizeof(prep));
        prep.orig_description = key->description;
        prep.data = data;
        prep.datalen = datalen;
        prep.quotalen = key->type->def_datalen;
        prep.expiry = TIME64_MAX;
        if (key->type->preparse) {
                ret = key->type->preparse(&prep);
                if (ret < 0)
                        goto error;
        }

        if (keyring) {
                ret = __key_link_lock(keyring, &key->index_key);
                if (ret < 0)
                        goto error;

                ret = __key_link_begin(keyring, &key->index_key, &edit);
                if (ret < 0)
                        goto error_link_end;

                if (keyring->restrict_link && keyring->restrict_link->check) {
                        struct key_restriction *keyres = keyring->restrict_link;

                        ret = keyres->check(keyring, key->type, &prep.payload,
                                            keyres->key);
                        if (ret < 0)
                                goto error_link_end;
                }
        }

        ret = __key_instantiate_and_link(key, &prep, keyring, authkey, &edit);

error_link_end:
        if (keyring)
                __key_link_end(keyring, &key->index_key, edit);

error:
        if (key->type->preparse)
                key->type->free_preparse(&prep);
        return ret;
}

EXPORT_SYMBOL(key_instantiate_and_link);

/**
 * key_reject_and_link - Negatively instantiate a key and link it into the keyring.
 * @key: The key to instantiate.
 * @timeout: The timeout on the negative key.
 * @error: The error to return when the key is hit.
 * @keyring: Keyring to create a link in on success (or NULL).
 * @authkey: The authorisation token permitting instantiation.
 *
 * Negatively instantiate a key that's in the uninstantiated state and, if
 * successful, set its timeout and stored error and link it in to the
 * destination keyring if one is supplied.  The key and any links to the key
 * will be automatically garbage collected after the timeout expires.
 *
 * Negative keys are used to rate limit repeated request_key() calls by causing
 * them to return the stored error code (typically ENOKEY) until the negative
 * key expires.
 *
 * If successful, 0 is returned, the authorisation token is revoked and anyone
 * waiting for the key is woken up.  If the key was already instantiated,
 * -EBUSY will be returned.
 */
int key_reject_and_link(struct key *key,
                        unsigned timeout,
                        unsigned error,
                        struct key *keyring,
                        struct key *authkey)
{
        struct assoc_array_edit *edit = NULL;
        int ret, awaken, link_ret = 0;

        key_check(key);
        key_check(keyring);

        awaken = 0;
        ret = -EBUSY;

        if (keyring) {
                if (keyring->restrict_link)
                        return -EPERM;

                link_ret = __key_link_lock(keyring, &key->index_key);
                if (link_ret == 0) {
                        link_ret = __key_link_begin(keyring, &key->index_key, &edit);
                        if (link_ret < 0)
                                __key_link_end(keyring, &key->index_key, edit);
                }
        }

        mutex_lock(&key_construction_mutex);

        /* can't instantiate twice */
        if (key->state == KEY_IS_UNINSTANTIATED) {
                /* mark the key as being negatively instantiated */
                atomic_inc(&key->user->nikeys);
                mark_key_instantiated(key, -error);
                notify_key(key, NOTIFY_KEY_INSTANTIATED, -error);
                key_set_expiry(key, ktime_get_real_seconds() + timeout);

                if (test_and_clear_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags))
                        awaken = 1;

                ret = 0;

                /* and link it into the destination keyring */
                if (keyring && link_ret == 0)
                        __key_link(keyring, key, &edit);

                /* disable the authorisation key */
                if (authkey)
                        key_invalidate(authkey);
        }

        mutex_unlock(&key_construction_mutex);

        if (keyring && link_ret == 0)
                __key_link_end(keyring, &key->index_key, edit);

        /* wake up anyone waiting for a key to be constructed */
        if (awaken)
                wake_up_bit(&key->flags, KEY_FLAG_USER_CONSTRUCT);

        return ret == 0 ? link_ret : ret;
}
EXPORT_SYMBOL(key_reject_and_link);

/**
 * key_put - Discard a reference to a key.
 * @key: The key to discard a reference from.
 *
 * Discard a reference to a key, and when all the references are gone, we
 * schedule the cleanup task to come and pull it out of the tree in process
 * context at some later time.
 */
void key_put(struct key *key)
{
        if (key) {
                key_check(key);

                if (refcount_dec_and_test(&key->usage)) {
                        unsigned long flags;

                        /* deal with the user's key tracking and quota */
                        if (test_bit(KEY_FLAG_IN_QUOTA, &key->flags)) {
                                spin_lock_irqsave(&key->user->lock, flags);
                                key->user->qnkeys--;
                                key->user->qnbytes -= key->quotalen;
                                spin_unlock_irqrestore(&key->user->lock, flags);
                        }
                        schedule_work(&key_gc_work);
                }
        }
}
EXPORT_SYMBOL(key_put);

/*
 * Find a key by its serial number.
 */
struct key *key_lookup(key_serial_t id)
{
        struct rb_node *n;
        struct key *key;

        spin_lock(&key_serial_lock);

        /* search the tree for the specified key */
        n = key_serial_tree.rb_node;
        while (n) {
                key = rb_entry(n, struct key, serial_node);

                if (id < key->serial)
                        n = n->rb_left;
                else if (id > key->serial)
                        n = n->rb_right;
                else
                        goto found;
        }

not_found:
        key = ERR_PTR(-ENOKEY);
        goto error;

found:
        /* A key is allowed to be looked up only if someone still owns a
         * reference to it - otherwise it's awaiting the gc.
         */
        if (!refcount_inc_not_zero(&key->usage))
                goto not_found;

error:
        spin_unlock(&key_serial_lock);
        return key;
}
EXPORT_SYMBOL(key_lookup);

/*
 * Find and lock the specified key type against removal.
 *
 * We return with the sem read-locked if successful.  If the type wasn't
 * available -ENOKEY is returned instead.
 */
struct key_type *key_type_lookup(const char *type)
{
        struct key_type *ktype;

        down_read(&key_types_sem);

        /* look up the key type to see if it's one of the registered kernel
         * types */
        list_for_each_entry(ktype, &key_types_list, link) {
                if (strcmp(ktype->name, type) == 0)
                        goto found_kernel_type;
        }

        up_read(&key_types_sem);
        ktype = ERR_PTR(-ENOKEY);

found_kernel_type:
        return ktype;
}

void key_set_timeout(struct key *key, unsigned timeout)
{
        time64_t expiry = TIME64_MAX;

        /* make the changes with the locks held to prevent races */
        down_write(&key->sem);

        if (timeout > 0)
                expiry = ktime_get_real_seconds() + timeout;
        key_set_expiry(key, expiry);

        up_write(&key->sem);
}
EXPORT_SYMBOL_GPL(key_set_timeout);

/*
 * Unlock a key type locked by key_type_lookup().
 */
void key_type_put(struct key_type *ktype)
{
        up_read(&key_types_sem);
}

/*
 * Attempt to update an existing key.
 *
 * The key is given to us with an incremented refcount that we need to discard
 * if we get an error.
 */
static inline key_ref_t __key_update(key_ref_t key_ref,
                                     struct key_preparsed_payload *prep)
{
        struct key *key = key_ref_to_ptr(key_ref);
        int ret;

        /* need write permission on the key to update it */
        ret = key_permission(key_ref, KEY_NEED_WRITE);
        if (ret < 0)
                goto error;

        ret = -EEXIST;
        if (!key->type->update)
                goto error;

        down_write(&key->sem);

        ret = key->type->update(key, prep);
        if (ret == 0) {
                /* Updating a negative key positively instantiates it */
                mark_key_instantiated(key, 0);
                notify_key(key, NOTIFY_KEY_UPDATED, 0);
        }

        up_write(&key->sem);

        if (ret < 0)
                goto error;
out:
        return key_ref;

error:
        key_put(key);
        key_ref = ERR_PTR(ret);
        goto out;
}

/*
 * Create or potentially update a key. The combined logic behind
 * key_create_or_update() and key_create()
 */
static key_ref_t __key_create_or_update(key_ref_t keyring_ref,
                                        const char *type,
                                        const char *description,
                                        const void *payload,
                                        size_t plen,
                                        key_perm_t perm,
                                        unsigned long flags,
                                        bool allow_update)
{
        struct keyring_index_key index_key = {
                .description        = description,
        };
        struct key_preparsed_payload prep;
        struct assoc_array_edit *edit = NULL;
        const struct cred *cred = current_cred();
        struct key *keyring, *key = NULL;
        key_ref_t key_ref;
        int ret;
        struct key_restriction *restrict_link = NULL;

        /* look up the key type to see if it's one of the registered kernel
         * types */
        index_key.type = key_type_lookup(type);
        if (IS_ERR(index_key.type)) {
                key_ref = ERR_PTR(-ENODEV);
                goto error;
        }

        key_ref = ERR_PTR(-EINVAL);
        if (!index_key.type->instantiate ||
            (!index_key.description && !index_key.type->preparse))
                goto error_put_type;

        keyring = key_ref_to_ptr(keyring_ref);

        key_check(keyring);

        if (!(flags & KEY_ALLOC_BYPASS_RESTRICTION))
                restrict_link = keyring->restrict_link;

        key_ref = ERR_PTR(-ENOTDIR);
        if (keyring->type != &key_type_keyring)
                goto error_put_type;

        memset(&prep, 0, sizeof(prep));
        prep.orig_description = description;
        prep.data = payload;
        prep.datalen = plen;
        prep.quotalen = index_key.type->def_datalen;
        prep.expiry = TIME64_MAX;
        if (index_key.type->preparse) {
                ret = index_key.type->preparse(&prep);
                if (ret < 0) {
                        key_ref = ERR_PTR(ret);
                        goto error_free_prep;
                }
                if (!index_key.description)
                        index_key.description = prep.description;
                key_ref = ERR_PTR(-EINVAL);
                if (!index_key.description)
                        goto error_free_prep;
        }
        index_key.desc_len = strlen(index_key.description);
        key_set_index_key(&index_key);

        ret = __key_link_lock(keyring, &index_key);
        if (ret < 0) {
                key_ref = ERR_PTR(ret);
                goto error_free_prep;
        }

        ret = __key_link_begin(keyring, &index_key, &edit);
        if (ret < 0) {
                key_ref = ERR_PTR(ret);
                goto error_link_end;
        }

        if (restrict_link && restrict_link->check) {
                ret = restrict_link->check(keyring, index_key.type,
                                           &prep.payload, restrict_link->key);
                if (ret < 0) {
                        key_ref = ERR_PTR(ret);
                        goto error_link_end;
                }
        }

        /* if we're going to allocate a new key, we're going to have
         * to modify the keyring */
        ret = key_permission(keyring_ref, KEY_NEED_WRITE);
        if (ret < 0) {
                key_ref = ERR_PTR(ret);
                goto error_link_end;
        }

        /* if it's requested and possible to update this type of key, search
         * for an existing key of the same type and description in the
         * destination keyring and update that instead if possible
         */
        if (allow_update) {
                if (index_key.type->update) {
                        key_ref = find_key_to_update(keyring_ref, &index_key);
                        if (key_ref)
                                goto found_matching_key;
                }
        } else {
                key_ref = find_key_to_update(keyring_ref, &index_key);
                if (key_ref) {
                        key_ref_put(key_ref);
                        key_ref = ERR_PTR(-EEXIST);
                        goto error_link_end;
                }
        }

        /* if the client doesn't provide, decide on the permissions we want */
        if (perm == KEY_PERM_UNDEF) {
                perm = KEY_POS_VIEW | KEY_POS_SEARCH | KEY_POS_LINK | KEY_POS_SETATTR;
                perm |= KEY_USR_VIEW;

                if (index_key.type->read)
                        perm |= KEY_POS_READ;

                if (index_key.type == &key_type_keyring ||
                    index_key.type->update)
                        perm |= KEY_POS_WRITE;
        }

        /* allocate a new key */
        key = key_alloc(index_key.type, index_key.description,
                        cred->fsuid, cred->fsgid, cred, perm, flags, NULL);
        if (IS_ERR(key)) {
                key_ref = ERR_CAST(key);
                goto error_link_end;
        }

        /* instantiate it and link it into the target keyring */
        ret = __key_instantiate_and_link(key, &prep, keyring, NULL, &edit);
        if (ret < 0) {
                key_put(key);
                key_ref = ERR_PTR(ret);
                goto error_link_end;
        }

        security_key_post_create_or_update(keyring, key, payload, plen, flags,
                                           true);

        key_ref = make_key_ref(key, is_key_possessed(keyring_ref));

error_link_end:
        __key_link_end(keyring, &index_key, edit);
error_free_prep:
        if (index_key.type->preparse)
                index_key.type->free_preparse(&prep);
error_put_type:
        key_type_put(index_key.type);
error:
        return key_ref;

 found_matching_key:
        /* we found a matching key, so we're going to try to update it
         * - we can drop the locks first as we have the key pinned
         */
        __key_link_end(keyring, &index_key, edit);

        key = key_ref_to_ptr(key_ref);
        if (test_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags)) {
                ret = wait_for_key_construction(key, true);
                if (ret < 0) {
                        key_ref_put(key_ref);
                        key_ref = ERR_PTR(ret);
                        goto error_free_prep;
                }
        }

        key_ref = __key_update(key_ref, &prep);

        if (!IS_ERR(key_ref))
                security_key_post_create_or_update(keyring, key, payload, plen,
                                                   flags, false);

        goto error_free_prep;
}

/**
 * key_create_or_update - Update or create and instantiate a key.
 * @keyring_ref: A pointer to the destination keyring with possession flag.
 * @type: The type of key.
 * @description: The searchable description for the key.
 * @payload: The data to use to instantiate or update the key.
 * @plen: The length of @payload.
 * @perm: The permissions mask for a new key.
 * @flags: The quota flags for a new key.
 *
 * Search the destination keyring for a key of the same description and if one
 * is found, update it, otherwise create and instantiate a new one and create a
 * link to it from that keyring.
 *
 * If perm is KEY_PERM_UNDEF then an appropriate key permissions mask will be
 * concocted.
 *
 * Returns a pointer to the new key if successful, -ENODEV if the key type
 * wasn't available, -ENOTDIR if the keyring wasn't a keyring, -EACCES if the
 * caller isn't permitted to modify the keyring or the LSM did not permit
 * creation of the key.
 *
 * On success, the possession flag from the keyring ref will be tacked on to
 * the key ref before it is returned.
 */
key_ref_t key_create_or_update(key_ref_t keyring_ref,
                               const char *type,
                               const char *description,
                               const void *payload,
                               size_t plen,
                               key_perm_t perm,
                               unsigned long flags)
{
        return __key_create_or_update(keyring_ref, type, description, payload,
                                      plen, perm, flags, true);
}
EXPORT_SYMBOL(key_create_or_update);

/**
 * key_create - Create and instantiate a key.
 * @keyring_ref: A pointer to the destination keyring with possession flag.
 * @type: The type of key.
 * @description: The searchable description for the key.
 * @payload: The data to use to instantiate or update the key.
 * @plen: The length of @payload.
 * @perm: The permissions mask for a new key.
 * @flags: The quota flags for a new key.
 *
 * Create and instantiate a new key and link to it from the destination keyring.
 *
 * If perm is KEY_PERM_UNDEF then an appropriate key permissions mask will be
 * concocted.
 *
 * Returns a pointer to the new key if successful, -EEXIST if a key with the
 * same description already exists, -ENODEV if the key type wasn't available,
 * -ENOTDIR if the keyring wasn't a keyring, -EACCES if the caller isn't
 * permitted to modify the keyring or the LSM did not permit creation of the
 * key.
 *
 * On success, the possession flag from the keyring ref will be tacked on to
 * the key ref before it is returned.
 */
key_ref_t key_create(key_ref_t keyring_ref,
                     const char *type,
                     const char *description,
                     const void *payload,
                     size_t plen,
                     key_perm_t perm,
                     unsigned long flags)
{
        return __key_create_or_update(keyring_ref, type, description, payload,
                                      plen, perm, flags, false);
}
EXPORT_SYMBOL(key_create);

/**
 * key_update - Update a key's contents.
 * @key_ref: The pointer (plus possession flag) to the key.
 * @payload: The data to be used to update the key.
 * @plen: The length of @payload.
 *
 * Attempt to update the contents of a key with the given payload data.  The
 * caller must be granted Write permission on the key.  Negative keys can be
 * instantiated by this method.
 *
 * Returns 0 on success, -EACCES if not permitted and -EOPNOTSUPP if the key
 * type does not support updating.  The key type may return other errors.
 */
int key_update(key_ref_t key_ref, const void *payload, size_t plen)
{
        struct key_preparsed_payload prep;
        struct key *key = key_ref_to_ptr(key_ref);
        int ret;

        key_check(key);

        /* the key must be writable */
        ret = key_permission(key_ref, KEY_NEED_WRITE);
        if (ret < 0)
                return ret;

        /* attempt to update it if supported */
        if (!key->type->update)
                return -EOPNOTSUPP;

        memset(&prep, 0, sizeof(prep));
        prep.data = payload;
        prep.datalen = plen;
        prep.quotalen = key->type->def_datalen;
        prep.expiry = TIME64_MAX;
        if (key->type->preparse) {
                ret = key->type->preparse(&prep);
                if (ret < 0)
                        goto error;
        }

        down_write(&key->sem);

        ret = key->type->update(key, &prep);
        if (ret == 0) {
                /* Updating a negative key positively instantiates it */
                mark_key_instantiated(key, 0);
                notify_key(key, NOTIFY_KEY_UPDATED, 0);
        }

        up_write(&key->sem);

error:
        if (key->type->preparse)
                key->type->free_preparse(&prep);
        return ret;
}
EXPORT_SYMBOL(key_update);

/**
 * key_revoke - Revoke a key.
 * @key: The key to be revoked.
 *
 * Mark a key as being revoked and ask the type to free up its resources.  The
 * revocation timeout is set and the key and all its links will be
 * automatically garbage collected after key_gc_delay amount of time if they
 * are not manually dealt with first.
 */
void key_revoke(struct key *key)
{
        time64_t time;

        key_check(key);

        /* make sure no one's trying to change or use the key when we mark it
         * - we tell lockdep that we might nest because we might be revoking an
         *   authorisation key whilst holding the sem on a key we've just
         *   instantiated
         */
        down_write_nested(&key->sem, 1);
        if (!test_and_set_bit(KEY_FLAG_REVOKED, &key->flags)) {
                notify_key(key, NOTIFY_KEY_REVOKED, 0);
                if (key->type->revoke)
                        key->type->revoke(key);

                /* set the death time to no more than the expiry time */
                time = ktime_get_real_seconds();
                if (key->revoked_at == 0 || key->revoked_at > time) {
                        key->revoked_at = time;
                        key_schedule_gc(key->revoked_at + key_gc_delay);
                }
        }

        up_write(&key->sem);
}
EXPORT_SYMBOL(key_revoke);

/**
 * key_invalidate - Invalidate a key.
 * @key: The key to be invalidated.
 *
 * Mark a key as being invalidated and have it cleaned up immediately.  The key
 * is ignored by all searches and other operations from this point.
 */
void key_invalidate(struct key *key)
{
        kenter("%d", key_serial(key));

        key_check(key);

        if (!test_bit(KEY_FLAG_INVALIDATED, &key->flags)) {
                down_write_nested(&key->sem, 1);
                if (!test_and_set_bit(KEY_FLAG_INVALIDATED, &key->flags)) {
                        notify_key(key, NOTIFY_KEY_INVALIDATED, 0);
                        key_schedule_gc_links();
                }
                up_write(&key->sem);
        }
}
EXPORT_SYMBOL(key_invalidate);

/**
 * generic_key_instantiate - Simple instantiation of a key from preparsed data
 * @key: The key to be instantiated
 * @prep: The preparsed data to load.
 *
 * Instantiate a key from preparsed data.  We assume we can just copy the data
 * in directly and clear the old pointers.
 *
 * This can be pointed to directly by the key type instantiate op pointer.
 */
int generic_key_instantiate(struct key *key, struct key_preparsed_payload *prep)
{
        int ret;

        pr_devel("==>%s()\n", __func__);

        ret = key_payload_reserve(key, prep->quotalen);
        if (ret == 0) {
                rcu_assign_keypointer(key, prep->payload.data[0]);
                key->payload.data[1] = prep->payload.data[1];
                key->payload.data[2] = prep->payload.data[2];
                key->payload.data[3] = prep->payload.data[3];
                prep->payload.data[0] = NULL;
                prep->payload.data[1] = NULL;
                prep->payload.data[2] = NULL;
                prep->payload.data[3] = NULL;
        }
        pr_devel("<==%s() = %d\n", __func__, ret);
        return ret;
}
EXPORT_SYMBOL(generic_key_instantiate);

/**
 * register_key_type - Register a type of key.
 * @ktype: The new key type.
 *
 * Register a new key type.
 *
 * Returns 0 on success or -EEXIST if a type of this name already exists.
 */
int register_key_type(struct key_type *ktype)
{
        struct key_type *p;
        int ret;

        memset(&ktype->lock_class, 0, sizeof(ktype->lock_class));

        ret = -EEXIST;
        down_write(&key_types_sem);

        /* disallow key types with the same name */
        list_for_each_entry(p, &key_types_list, link) {
                if (strcmp(p->name, ktype->name) == 0)
                        goto out;
        }

        /* store the type */
        list_add(&ktype->link, &key_types_list);

        pr_notice("Key type %s registered\n", ktype->name);
        ret = 0;

out:
        up_write(&key_types_sem);
        return ret;
}
EXPORT_SYMBOL(register_key_type);

/**
 * unregister_key_type - Unregister a type of key.
 * @ktype: The key type.
 *
 * Unregister a key type and mark all the extant keys of this type as dead.
 * Those keys of this type are then destroyed to get rid of their payloads and
 * they and their links will be garbage collected as soon as possible.
 */
void unregister_key_type(struct key_type *ktype)
{
        down_write(&key_types_sem);
        list_del_init(&ktype->link);
        downgrade_write(&key_types_sem);
        key_gc_keytype(ktype);
        pr_notice("Key type %s unregistered\n", ktype->name);
        up_read(&key_types_sem);
}
EXPORT_SYMBOL(unregister_key_type);

/*
 * Initialise the key management state.
 */
void __init key_init(void)
{
        /* allocate a slab in which we can store keys */
        key_jar = kmem_cache_create("key_jar", sizeof(struct key),
                        0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);

        /* add the special key types */
        list_add_tail(&key_type_keyring.link, &key_types_list);
        list_add_tail(&key_type_dead.link, &key_types_list);
        list_add_tail(&key_type_user.link, &key_types_list);
        list_add_tail(&key_type_logon.link, &key_types_list);

        /* record the root user tracking */
        rb_link_node(&root_key_user.node,
                     NULL,
                     &key_user_tree.rb_node);

        rb_insert_color(&root_key_user.node,
                        &key_user_tree);
}






















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  131 
  132 
























    7 






























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
#ifndef _BTRFS_CTREE_H_
#define _BTRFS_CTREE_H_

#include <linux/btrfs.h>
#include <linux/types.h>
#ifdef __KERNEL__
#include <linux/stddef.h>
#else
#include <stddef.h>
#endif

/* ASCII for _BHRfS_M, no terminating nul */
#define BTRFS_MAGIC 0x4D5F53665248425FULL

#define BTRFS_MAX_LEVEL 8

/*
 * We can actually store much bigger names, but lets not confuse the rest of
 * linux.
 */
#define BTRFS_NAME_LEN 255

/*
 * Theoretical limit is larger, but we keep this down to a sane value. That
 * should limit greatly the possibility of collisions on inode ref items.
 */
#define BTRFS_LINK_MAX 65535U

/*
 * This header contains the structure definitions and constants used
 * by file system objects that can be retrieved using
 * the BTRFS_IOC_SEARCH_TREE ioctl.  That means basically anything that
 * is needed to describe a leaf node's key or item contents.
 */

/* holds pointers to all of the tree roots */
#define BTRFS_ROOT_TREE_OBJECTID 1ULL

/* stores information about which extents are in use, and reference counts */
#define BTRFS_EXTENT_TREE_OBJECTID 2ULL

/*
 * chunk tree stores translations from logical -> physical block numbering
 * the super block points to the chunk tree
 */
#define BTRFS_CHUNK_TREE_OBJECTID 3ULL

/*
 * stores information about which areas of a given device are in use.
 * one per device.  The tree of tree roots points to the device tree
 */
#define BTRFS_DEV_TREE_OBJECTID 4ULL

/* one per subvolume, storing files and directories */
#define BTRFS_FS_TREE_OBJECTID 5ULL

/* directory objectid inside the root tree */
#define BTRFS_ROOT_TREE_DIR_OBJECTID 6ULL

/* holds checksums of all the data extents */
#define BTRFS_CSUM_TREE_OBJECTID 7ULL

/* holds quota configuration and tracking */
#define BTRFS_QUOTA_TREE_OBJECTID 8ULL

/* for storing items that use the BTRFS_UUID_KEY* types */
#define BTRFS_UUID_TREE_OBJECTID 9ULL

/* tracks free space in block groups. */
#define BTRFS_FREE_SPACE_TREE_OBJECTID 10ULL

/* Holds the block group items for extent tree v2. */
#define BTRFS_BLOCK_GROUP_TREE_OBJECTID 11ULL

/* Tracks RAID stripes in block groups. */
#define BTRFS_RAID_STRIPE_TREE_OBJECTID 12ULL

/* device stats in the device tree */
#define BTRFS_DEV_STATS_OBJECTID 0ULL

/* for storing balance parameters in the root tree */
#define BTRFS_BALANCE_OBJECTID -4ULL

/* orphan objectid for tracking unlinked/truncated files */
#define BTRFS_ORPHAN_OBJECTID -5ULL

/* does write ahead logging to speed up fsyncs */
#define BTRFS_TREE_LOG_OBJECTID -6ULL
#define BTRFS_TREE_LOG_FIXUP_OBJECTID -7ULL

/* for space balancing */
#define BTRFS_TREE_RELOC_OBJECTID -8ULL
#define BTRFS_DATA_RELOC_TREE_OBJECTID -9ULL

/*
 * extent checksums all have this objectid
 * this allows them to share the logging tree
 * for fsyncs
 */
#define BTRFS_EXTENT_CSUM_OBJECTID -10ULL

/* For storing free space cache */
#define BTRFS_FREE_SPACE_OBJECTID -11ULL

/*
 * The inode number assigned to the special inode for storing
 * free ino cache
 */
#define BTRFS_FREE_INO_OBJECTID -12ULL

/* dummy objectid represents multiple objectids */
#define BTRFS_MULTIPLE_OBJECTIDS -255ULL

/*
 * All files have objectids in this range.
 */
#define BTRFS_FIRST_FREE_OBJECTID 256ULL
#define BTRFS_LAST_FREE_OBJECTID -256ULL
#define BTRFS_FIRST_CHUNK_TREE_OBJECTID 256ULL


/*
 * the device items go into the chunk tree.  The key is in the form
 * [ 1 BTRFS_DEV_ITEM_KEY device_id ]
 */
#define BTRFS_DEV_ITEMS_OBJECTID 1ULL

#define BTRFS_BTREE_INODE_OBJECTID 1

#define BTRFS_EMPTY_SUBVOL_DIR_OBJECTID 2

#define BTRFS_DEV_REPLACE_DEVID 0ULL

/*
 * inode items have the data typically returned from stat and store other
 * info about object characteristics.  There is one for every file and dir in
 * the FS
 */
#define BTRFS_INODE_ITEM_KEY                1
#define BTRFS_INODE_REF_KEY                12
#define BTRFS_INODE_EXTREF_KEY                13
#define BTRFS_XATTR_ITEM_KEY                24

/*
 * fs verity items are stored under two different key types on disk.
 * The descriptor items:
 * [ inode objectid, BTRFS_VERITY_DESC_ITEM_KEY, offset ]
 *
 * At offset 0, we store a btrfs_verity_descriptor_item which tracks the size
 * of the descriptor item and some extra data for encryption.
 * Starting at offset 1, these hold the generic fs verity descriptor.  The
 * latter are opaque to btrfs, we just read and write them as a blob for the
 * higher level verity code.  The most common descriptor size is 256 bytes.
 *
 * The merkle tree items:
 * [ inode objectid, BTRFS_VERITY_MERKLE_ITEM_KEY, offset ]
 *
 * These also start at offset 0, and correspond to the merkle tree bytes.  When
 * fsverity asks for page 0 of the merkle tree, we pull up one page starting at
 * offset 0 for this key type.  These are also opaque to btrfs, we're blindly
 * storing whatever fsverity sends down.
 */
#define BTRFS_VERITY_DESC_ITEM_KEY        36
#define BTRFS_VERITY_MERKLE_ITEM_KEY        37

#define BTRFS_ORPHAN_ITEM_KEY                48
/* reserve 2-15 close to the inode for later flexibility */

/*
 * dir items are the name -> inode pointers in a directory.  There is one
 * for every name in a directory.  BTRFS_DIR_LOG_ITEM_KEY is no longer used
 * but it's still defined here for documentation purposes and to help avoid
 * having its numerical value reused in the future.
 */
#define BTRFS_DIR_LOG_ITEM_KEY  60
#define BTRFS_DIR_LOG_INDEX_KEY 72
#define BTRFS_DIR_ITEM_KEY        84
#define BTRFS_DIR_INDEX_KEY        96
/*
 * extent data is for file data
 */
#define BTRFS_EXTENT_DATA_KEY        108

/*
 * extent csums are stored in a separate tree and hold csums for
 * an entire extent on disk.
 */
#define BTRFS_EXTENT_CSUM_KEY        128

/*
 * root items point to tree roots.  They are typically in the root
 * tree used by the super block to find all the other trees
 */
#define BTRFS_ROOT_ITEM_KEY        132

/*
 * root backrefs tie subvols and snapshots to the directory entries that
 * reference them
 */
#define BTRFS_ROOT_BACKREF_KEY        144

/*
 * root refs make a fast index for listing all of the snapshots and
 * subvolumes referenced by a given root.  They point directly to the
 * directory item in the root that references the subvol
 */
#define BTRFS_ROOT_REF_KEY        156

/*
 * extent items are in the extent map tree.  These record which blocks
 * are used, and how many references there are to each block
 */
#define BTRFS_EXTENT_ITEM_KEY        168

/*
 * The same as the BTRFS_EXTENT_ITEM_KEY, except it's metadata we already know
 * the length, so we save the level in key->offset instead of the length.
 */
#define BTRFS_METADATA_ITEM_KEY        169

/*
 * Special inline ref key which stores the id of the subvolume which originally
 * created the extent. This subvolume owns the extent permanently from the
 * perspective of simple quotas. Needed to know which subvolume to free quota
 * usage from when the extent is deleted.
 *
 * Stored as an inline ref rather to avoid wasting space on a separate item on
 * top of the existing extent item. However, unlike the other inline refs,
 * there is one one owner ref per extent rather than one per extent.
 *
 * Because of this, it goes at the front of the list of inline refs, and thus
 * must have a lower type value than any other inline ref type (to satisfy the
 * disk format rule that inline refs have non-decreasing type).
 */
#define BTRFS_EXTENT_OWNER_REF_KEY        172

#define BTRFS_TREE_BLOCK_REF_KEY        176

#define BTRFS_EXTENT_DATA_REF_KEY        178

/*
 * Obsolete key. Defintion removed in 6.6, value may be reused in the future.
 *
 * #define BTRFS_EXTENT_REF_V0_KEY        180
 */

#define BTRFS_SHARED_BLOCK_REF_KEY        182

#define BTRFS_SHARED_DATA_REF_KEY        184

/*
 * block groups give us hints into the extent allocation trees.  Which
 * blocks are free etc etc
 */
#define BTRFS_BLOCK_GROUP_ITEM_KEY 192

/*
 * Every block group is represented in the free space tree by a free space info
 * item, which stores some accounting information. It is keyed on
 * (block_group_start, FREE_SPACE_INFO, block_group_length).
 */
#define BTRFS_FREE_SPACE_INFO_KEY 198

/*
 * A free space extent tracks an extent of space that is free in a block group.
 * It is keyed on (start, FREE_SPACE_EXTENT, length).
 */
#define BTRFS_FREE_SPACE_EXTENT_KEY 199

/*
 * When a block group becomes very fragmented, we convert it to use bitmaps
 * instead of extents. A free space bitmap is keyed on
 * (start, FREE_SPACE_BITMAP, length); the corresponding item is a bitmap with
 * (length / sectorsize) bits.
 */
#define BTRFS_FREE_SPACE_BITMAP_KEY 200

#define BTRFS_DEV_EXTENT_KEY        204
#define BTRFS_DEV_ITEM_KEY        216
#define BTRFS_CHUNK_ITEM_KEY        228

#define BTRFS_RAID_STRIPE_KEY        230

/*
 * Records the overall state of the qgroups.
 * There's only one instance of this key present,
 * (0, BTRFS_QGROUP_STATUS_KEY, 0)
 */
#define BTRFS_QGROUP_STATUS_KEY         240
/*
 * Records the currently used space of the qgroup.
 * One key per qgroup, (0, BTRFS_QGROUP_INFO_KEY, qgroupid).
 */
#define BTRFS_QGROUP_INFO_KEY           242
/*
 * Contains the user configured limits for the qgroup.
 * One key per qgroup, (0, BTRFS_QGROUP_LIMIT_KEY, qgroupid).
 */
#define BTRFS_QGROUP_LIMIT_KEY          244
/*
 * Records the child-parent relationship of qgroups. For
 * each relation, 2 keys are present:
 * (childid, BTRFS_QGROUP_RELATION_KEY, parentid)
 * (parentid, BTRFS_QGROUP_RELATION_KEY, childid)
 */
#define BTRFS_QGROUP_RELATION_KEY       246

/*
 * Obsolete name, see BTRFS_TEMPORARY_ITEM_KEY.
 */
#define BTRFS_BALANCE_ITEM_KEY        248

/*
 * The key type for tree items that are stored persistently, but do not need to
 * exist for extended period of time. The items can exist in any tree.
 *
 * [subtype, BTRFS_TEMPORARY_ITEM_KEY, data]
 *
 * Existing items:
 *
 * - balance status item
 *   (BTRFS_BALANCE_OBJECTID, BTRFS_TEMPORARY_ITEM_KEY, 0)
 */
#define BTRFS_TEMPORARY_ITEM_KEY        248

/*
 * Obsolete name, see BTRFS_PERSISTENT_ITEM_KEY
 */
#define BTRFS_DEV_STATS_KEY                249

/*
 * The key type for tree items that are stored persistently and usually exist
 * for a long period, eg. filesystem lifetime. The item kinds can be status
 * information, stats or preference values. The item can exist in any tree.
 *
 * [subtype, BTRFS_PERSISTENT_ITEM_KEY, data]
 *
 * Existing items:
 *
 * - device statistics, store IO stats in the device tree, one key for all
 *   stats
 *   (BTRFS_DEV_STATS_OBJECTID, BTRFS_DEV_STATS_KEY, 0)
 */
#define BTRFS_PERSISTENT_ITEM_KEY        249

/*
 * Persistently stores the device replace state in the device tree.
 * The key is built like this: (0, BTRFS_DEV_REPLACE_KEY, 0).
 */
#define BTRFS_DEV_REPLACE_KEY        250

/*
 * Stores items that allow to quickly map UUIDs to something else.
 * These items are part of the filesystem UUID tree.
 * The key is built like this:
 * (UUID_upper_64_bits, BTRFS_UUID_KEY*, UUID_lower_64_bits).
 */
#if BTRFS_UUID_SIZE != 16
#error "UUID items require BTRFS_UUID_SIZE == 16!"
#endif
#define BTRFS_UUID_KEY_SUBVOL        251        /* for UUIDs assigned to subvols */
#define BTRFS_UUID_KEY_RECEIVED_SUBVOL        252        /* for UUIDs assigned to
                                                 * received subvols */

/*
 * string items are for debugging.  They just store a short string of
 * data in the FS
 */
#define BTRFS_STRING_ITEM_KEY        253

/* Maximum metadata block size (nodesize) */
#define BTRFS_MAX_METADATA_BLOCKSIZE                        65536

/* 32 bytes in various csum fields */
#define BTRFS_CSUM_SIZE 32

/* csum types */
enum btrfs_csum_type {
        BTRFS_CSUM_TYPE_CRC32        = 0,
        BTRFS_CSUM_TYPE_XXHASH        = 1,
        BTRFS_CSUM_TYPE_SHA256        = 2,
        BTRFS_CSUM_TYPE_BLAKE2        = 3,
};

/*
 * flags definitions for directory entry item type
 *
 * Used by:
 * struct btrfs_dir_item.type
 *
 * Values 0..7 must match common file type values in fs_types.h.
 */
#define BTRFS_FT_UNKNOWN        0
#define BTRFS_FT_REG_FILE        1
#define BTRFS_FT_DIR                2
#define BTRFS_FT_CHRDEV                3
#define BTRFS_FT_BLKDEV                4
#define BTRFS_FT_FIFO                5
#define BTRFS_FT_SOCK                6
#define BTRFS_FT_SYMLINK        7
#define BTRFS_FT_XATTR                8
#define BTRFS_FT_MAX                9
/* Directory contains encrypted data */
#define BTRFS_FT_ENCRYPTED        0x80

static inline __u8 btrfs_dir_flags_to_ftype(__u8 flags)
{
        return flags & ~BTRFS_FT_ENCRYPTED;
}

/*
 * Inode flags
 */
#define BTRFS_INODE_NODATASUM                (1U << 0)
#define BTRFS_INODE_NODATACOW                (1U << 1)
#define BTRFS_INODE_READONLY                (1U << 2)
#define BTRFS_INODE_NOCOMPRESS                (1U << 3)
#define BTRFS_INODE_PREALLOC                (1U << 4)
#define BTRFS_INODE_SYNC                (1U << 5)
#define BTRFS_INODE_IMMUTABLE                (1U << 6)
#define BTRFS_INODE_APPEND                (1U << 7)
#define BTRFS_INODE_NODUMP                (1U << 8)
#define BTRFS_INODE_NOATIME                (1U << 9)
#define BTRFS_INODE_DIRSYNC                (1U << 10)
#define BTRFS_INODE_COMPRESS                (1U << 11)

#define BTRFS_INODE_ROOT_ITEM_INIT        (1U << 31)

#define BTRFS_INODE_FLAG_MASK                                                \
        (BTRFS_INODE_NODATASUM |                                        \
         BTRFS_INODE_NODATACOW |                                        \
         BTRFS_INODE_READONLY |                                                \
         BTRFS_INODE_NOCOMPRESS |                                        \
         BTRFS_INODE_PREALLOC |                                                \
         BTRFS_INODE_SYNC |                                                \
         BTRFS_INODE_IMMUTABLE |                                        \
         BTRFS_INODE_APPEND |                                                \
         BTRFS_INODE_NODUMP |                                                \
         BTRFS_INODE_NOATIME |                                                \
         BTRFS_INODE_DIRSYNC |                                                \
         BTRFS_INODE_COMPRESS |                                                \
         BTRFS_INODE_ROOT_ITEM_INIT)

#define BTRFS_INODE_RO_VERITY                (1U << 0)

#define BTRFS_INODE_RO_FLAG_MASK        (BTRFS_INODE_RO_VERITY)

/*
 * The key defines the order in the tree, and so it also defines (optimal)
 * block layout.
 *
 * objectid corresponds to the inode number.
 *
 * type tells us things about the object, and is a kind of stream selector.
 * so for a given inode, keys with type of 1 might refer to the inode data,
 * type of 2 may point to file data in the btree and type == 3 may point to
 * extents.
 *
 * offset is the starting byte offset for this key in the stream.
 *
 * btrfs_disk_key is in disk byte order.  struct btrfs_key is always
 * in cpu native order.  Otherwise they are identical and their sizes
 * should be the same (ie both packed)
 */
struct btrfs_disk_key {
        __le64 objectid;
        __u8 type;
        __le64 offset;
} __attribute__ ((__packed__));

struct btrfs_key {
        __u64 objectid;
        __u8 type;
        __u64 offset;
} __attribute__ ((__packed__));

/*
 * Every tree block (leaf or node) starts with this header.
 */
struct btrfs_header {
        /* These first four must match the super block */
        __u8 csum[BTRFS_CSUM_SIZE];
        /* FS specific uuid */
        __u8 fsid[BTRFS_FSID_SIZE];
        /* Which block this node is supposed to live in */
        __le64 bytenr;
        __le64 flags;

        /* Allowed to be different from the super from here on down */
        __u8 chunk_tree_uuid[BTRFS_UUID_SIZE];
        __le64 generation;
        __le64 owner;
        __le32 nritems;
        __u8 level;
} __attribute__ ((__packed__));

/*
 * This is a very generous portion of the super block, giving us room to
 * translate 14 chunks with 3 stripes each.
 */
#define BTRFS_SYSTEM_CHUNK_ARRAY_SIZE 2048

/*
 * Just in case we somehow lose the roots and are not able to mount, we store
 * an array of the roots from previous transactions in the super.
 */
#define BTRFS_NUM_BACKUP_ROOTS 4
struct btrfs_root_backup {
        __le64 tree_root;
        __le64 tree_root_gen;

        __le64 chunk_root;
        __le64 chunk_root_gen;

        __le64 extent_root;
        __le64 extent_root_gen;

        __le64 fs_root;
        __le64 fs_root_gen;

        __le64 dev_root;
        __le64 dev_root_gen;

        __le64 csum_root;
        __le64 csum_root_gen;

        __le64 total_bytes;
        __le64 bytes_used;
        __le64 num_devices;
        /* future */
        __le64 unused_64[4];

        __u8 tree_root_level;
        __u8 chunk_root_level;
        __u8 extent_root_level;
        __u8 fs_root_level;
        __u8 dev_root_level;
        __u8 csum_root_level;
        /* future and to align */
        __u8 unused_8[10];
} __attribute__ ((__packed__));

/*
 * A leaf is full of items. offset and size tell us where to find the item in
 * the leaf (relative to the start of the data area)
 */
struct btrfs_item {
        struct btrfs_disk_key key;
        __le32 offset;
        __le32 size;
} __attribute__ ((__packed__));

/*
 * Leaves have an item area and a data area:
 * [item0, item1....itemN] [free space] [dataN...data1, data0]
 *
 * The data is separate from the items to get the keys closer together during
 * searches.
 */
struct btrfs_leaf {
        struct btrfs_header header;
        struct btrfs_item items[];
} __attribute__ ((__packed__));

/*
 * All non-leaf blocks are nodes, they hold only keys and pointers to other
 * blocks.
 */
struct btrfs_key_ptr {
        struct btrfs_disk_key key;
        __le64 blockptr;
        __le64 generation;
} __attribute__ ((__packed__));

struct btrfs_node {
        struct btrfs_header header;
        struct btrfs_key_ptr ptrs[];
} __attribute__ ((__packed__));

struct btrfs_dev_item {
        /* the internal btrfs device id */
        __le64 devid;

        /* size of the device */
        __le64 total_bytes;

        /* bytes used */
        __le64 bytes_used;

        /* optimal io alignment for this device */
        __le32 io_align;

        /* optimal io width for this device */
        __le32 io_width;

        /* minimal io size for this device */
        __le32 sector_size;

        /* type and info about this device */
        __le64 type;

        /* expected generation for this device */
        __le64 generation;

        /*
         * starting byte of this partition on the device,
         * to allow for stripe alignment in the future
         */
        __le64 start_offset;

        /* grouping information for allocation decisions */
        __le32 dev_group;

        /* seek speed 0-100 where 100 is fastest */
        __u8 seek_speed;

        /* bandwidth 0-100 where 100 is fastest */
        __u8 bandwidth;

        /* btrfs generated uuid for this device */
        __u8 uuid[BTRFS_UUID_SIZE];

        /* uuid of FS who owns this device */
        __u8 fsid[BTRFS_UUID_SIZE];
} __attribute__ ((__packed__));

struct btrfs_stripe {
        __le64 devid;
        __le64 offset;
        __u8 dev_uuid[BTRFS_UUID_SIZE];
} __attribute__ ((__packed__));

struct btrfs_chunk {
        /* size of this chunk in bytes */
        __le64 length;

        /* objectid of the root referencing this chunk */
        __le64 owner;

        __le64 stripe_len;
        __le64 type;

        /* optimal io alignment for this chunk */
        __le32 io_align;

        /* optimal io width for this chunk */
        __le32 io_width;

        /* minimal io size for this chunk */
        __le32 sector_size;

        /* 2^16 stripes is quite a lot, a second limit is the size of a single
         * item in the btree
         */
        __le16 num_stripes;

        /* sub stripes only matter for raid10 */
        __le16 sub_stripes;
        struct btrfs_stripe stripe;
        /* additional stripes go here */
} __attribute__ ((__packed__));

/*
 * The super block basically lists the main trees of the FS.
 */
struct btrfs_super_block {
        /* The first 4 fields must match struct btrfs_header */
        __u8 csum[BTRFS_CSUM_SIZE];
        /* FS specific UUID, visible to user */
        __u8 fsid[BTRFS_FSID_SIZE];
        /* This block number */
        __le64 bytenr;
        __le64 flags;

        /* Allowed to be different from the btrfs_header from here own down */
        __le64 magic;
        __le64 generation;
        __le64 root;
        __le64 chunk_root;
        __le64 log_root;

        /*
         * This member has never been utilized since the very beginning, thus
         * it's always 0 regardless of kernel version.  We always use
         * generation + 1 to read log tree root.  So here we mark it deprecated.
         */
        __le64 __unused_log_root_transid;
        __le64 total_bytes;
        __le64 bytes_used;
        __le64 root_dir_objectid;
        __le64 num_devices;
        __le32 sectorsize;
        __le32 nodesize;
        __le32 __unused_leafsize;
        __le32 stripesize;
        __le32 sys_chunk_array_size;
        __le64 chunk_root_generation;
        __le64 compat_flags;
        __le64 compat_ro_flags;
        __le64 incompat_flags;
        __le16 csum_type;
        __u8 root_level;
        __u8 chunk_root_level;
        __u8 log_root_level;
        struct btrfs_dev_item dev_item;

        char label[BTRFS_LABEL_SIZE];

        __le64 cache_generation;
        __le64 uuid_tree_generation;

        /* The UUID written into btree blocks */
        __u8 metadata_uuid[BTRFS_FSID_SIZE];

        __u64 nr_global_roots;

        /* Future expansion */
        __le64 reserved[27];
        __u8 sys_chunk_array[BTRFS_SYSTEM_CHUNK_ARRAY_SIZE];
        struct btrfs_root_backup super_roots[BTRFS_NUM_BACKUP_ROOTS];

        /* Padded to 4096 bytes */
        __u8 padding[565];
} __attribute__ ((__packed__));

#define BTRFS_FREE_SPACE_EXTENT        1
#define BTRFS_FREE_SPACE_BITMAP        2

struct btrfs_free_space_entry {
        __le64 offset;
        __le64 bytes;
        __u8 type;
} __attribute__ ((__packed__));

struct btrfs_free_space_header {
        struct btrfs_disk_key location;
        __le64 generation;
        __le64 num_entries;
        __le64 num_bitmaps;
} __attribute__ ((__packed__));

struct btrfs_raid_stride {
        /* The id of device this raid extent lives on. */
        __le64 devid;
        /* The physical location on disk. */
        __le64 physical;
} __attribute__ ((__packed__));

struct btrfs_stripe_extent {
        /* An array of raid strides this stripe is composed of. */
        __DECLARE_FLEX_ARRAY(struct btrfs_raid_stride, strides);
} __attribute__ ((__packed__));

#define BTRFS_HEADER_FLAG_WRITTEN        (1ULL << 0)
#define BTRFS_HEADER_FLAG_RELOC                (1ULL << 1)

/* Super block flags */
/* Errors detected */
#define BTRFS_SUPER_FLAG_ERROR                (1ULL << 2)

#define BTRFS_SUPER_FLAG_SEEDING        (1ULL << 32)
#define BTRFS_SUPER_FLAG_METADUMP        (1ULL << 33)
#define BTRFS_SUPER_FLAG_METADUMP_V2        (1ULL << 34)
#define BTRFS_SUPER_FLAG_CHANGING_FSID        (1ULL << 35)
#define BTRFS_SUPER_FLAG_CHANGING_FSID_V2 (1ULL << 36)

/*
 * Those are temporaray flags utilized by btrfs-progs to do offline conversion.
 * They are rejected by kernel.
 * But still keep them all here to avoid conflicts.
 */
#define BTRFS_SUPER_FLAG_CHANGING_BG_TREE        (1ULL << 38)
#define BTRFS_SUPER_FLAG_CHANGING_DATA_CSUM        (1ULL << 39)
#define BTRFS_SUPER_FLAG_CHANGING_META_CSUM        (1ULL << 40)

/*
 * items in the extent btree are used to record the objectid of the
 * owner of the block and the number of references
 */

struct btrfs_extent_item {
        __le64 refs;
        __le64 generation;
        __le64 flags;
} __attribute__ ((__packed__));

struct btrfs_extent_item_v0 {
        __le32 refs;
} __attribute__ ((__packed__));


#define BTRFS_EXTENT_FLAG_DATA                (1ULL << 0)
#define BTRFS_EXTENT_FLAG_TREE_BLOCK        (1ULL << 1)

/* following flags only apply to tree blocks */

/* use full backrefs for extent pointers in the block */
#define BTRFS_BLOCK_FLAG_FULL_BACKREF        (1ULL << 8)

#define BTRFS_BACKREF_REV_MAX                256
#define BTRFS_BACKREF_REV_SHIFT                56
#define BTRFS_BACKREF_REV_MASK                (((u64)BTRFS_BACKREF_REV_MAX - 1) << \
                                         BTRFS_BACKREF_REV_SHIFT)

#define BTRFS_OLD_BACKREF_REV                0
#define BTRFS_MIXED_BACKREF_REV                1

/*
 * this flag is only used internally by scrub and may be changed at any time
 * it is only declared here to avoid collisions
 */
#define BTRFS_EXTENT_FLAG_SUPER                (1ULL << 48)

struct btrfs_tree_block_info {
        struct btrfs_disk_key key;
        __u8 level;
} __attribute__ ((__packed__));

struct btrfs_extent_data_ref {
        __le64 root;
        __le64 objectid;
        __le64 offset;
        __le32 count;
} __attribute__ ((__packed__));

struct btrfs_shared_data_ref {
        __le32 count;
} __attribute__ ((__packed__));

struct btrfs_extent_owner_ref {
        __le64 root_id;
} __attribute__ ((__packed__));

struct btrfs_extent_inline_ref {
        __u8 type;
        __le64 offset;
} __attribute__ ((__packed__));

/* dev extents record free space on individual devices.  The owner
 * field points back to the chunk allocation mapping tree that allocated
 * the extent.  The chunk tree uuid field is a way to double check the owner
 */
struct btrfs_dev_extent {
        __le64 chunk_tree;
        __le64 chunk_objectid;
        __le64 chunk_offset;
        __le64 length;
        __u8 chunk_tree_uuid[BTRFS_UUID_SIZE];
} __attribute__ ((__packed__));

struct btrfs_inode_ref {
        __le64 index;
        __le16 name_len;
        /* name goes here */
} __attribute__ ((__packed__));

struct btrfs_inode_extref {
        __le64 parent_objectid;
        __le64 index;
        __le16 name_len;
        __u8   name[];
        /* name goes here */
} __attribute__ ((__packed__));

struct btrfs_timespec {
        __le64 sec;
        __le32 nsec;
} __attribute__ ((__packed__));

struct btrfs_inode_item {
        /* nfs style generation number */
        __le64 generation;
        /* transid that last touched this inode */
        __le64 transid;
        __le64 size;
        __le64 nbytes;
        __le64 block_group;
        __le32 nlink;
        __le32 uid;
        __le32 gid;
        __le32 mode;
        __le64 rdev;
        __le64 flags;

        /* modification sequence number for NFS */
        __le64 sequence;

        /*
         * a little future expansion, for more than this we can
         * just grow the inode item and version it
         */
        __le64 reserved[4];
        struct btrfs_timespec atime;
        struct btrfs_timespec ctime;
        struct btrfs_timespec mtime;
        struct btrfs_timespec otime;
} __attribute__ ((__packed__));

struct btrfs_dir_log_item {
        __le64 end;
} __attribute__ ((__packed__));

struct btrfs_dir_item {
        struct btrfs_disk_key location;
        __le64 transid;
        __le16 data_len;
        __le16 name_len;
        __u8 type;
} __attribute__ ((__packed__));

#define BTRFS_ROOT_SUBVOL_RDONLY        (1ULL << 0)

/*
 * Internal in-memory flag that a subvolume has been marked for deletion but
 * still visible as a directory
 */
#define BTRFS_ROOT_SUBVOL_DEAD                (1ULL << 48)

struct btrfs_root_item {
        struct btrfs_inode_item inode;
        __le64 generation;
        __le64 root_dirid;
        __le64 bytenr;
        __le64 byte_limit;
        __le64 bytes_used;
        __le64 last_snapshot;
        __le64 flags;
        __le32 refs;
        struct btrfs_disk_key drop_progress;
        __u8 drop_level;
        __u8 level;

        /*
         * The following fields appear after subvol_uuids+subvol_times
         * were introduced.
         */

        /*
         * This generation number is used to test if the new fields are valid
         * and up to date while reading the root item. Every time the root item
         * is written out, the "generation" field is copied into this field. If
         * anyone ever mounted the fs with an older kernel, we will have
         * mismatching generation values here and thus must invalidate the
         * new fields. See btrfs_update_root and btrfs_find_last_root for
         * details.
         * the offset of generation_v2 is also used as the start for the memset
         * when invalidating the fields.
         */
        __le64 generation_v2;
        __u8 uuid[BTRFS_UUID_SIZE];
        __u8 parent_uuid[BTRFS_UUID_SIZE];
        __u8 received_uuid[BTRFS_UUID_SIZE];
        __le64 ctransid; /* updated when an inode changes */
        __le64 otransid; /* trans when created */
        __le64 stransid; /* trans when sent. non-zero for received subvol */
        __le64 rtransid; /* trans when received. non-zero for received subvol */
        struct btrfs_timespec ctime;
        struct btrfs_timespec otime;
        struct btrfs_timespec stime;
        struct btrfs_timespec rtime;
        __le64 reserved[8]; /* for future */
} __attribute__ ((__packed__));

/*
 * Btrfs root item used to be smaller than current size.  The old format ends
 * at where member generation_v2 is.
 */
static inline __u32 btrfs_legacy_root_item_size(void)
{
        return offsetof(struct btrfs_root_item, generation_v2);
}

/*
 * this is used for both forward and backward root refs
 */
struct btrfs_root_ref {
        __le64 dirid;
        __le64 sequence;
        __le16 name_len;
} __attribute__ ((__packed__));

struct btrfs_disk_balance_args {
        /*
         * profiles to operate on, single is denoted by
         * BTRFS_AVAIL_ALLOC_BIT_SINGLE
         */
        __le64 profiles;

        /*
         * usage filter
         * BTRFS_BALANCE_ARGS_USAGE with a single value means '0..N'
         * BTRFS_BALANCE_ARGS_USAGE_RANGE - range syntax, min..max
         */
        union {
                __le64 usage;
                struct {
                        __le32 usage_min;
                        __le32 usage_max;
                };
        };

        /* devid filter */
        __le64 devid;

        /* devid subset filter [pstart..pend) */
        __le64 pstart;
        __le64 pend;

        /* btrfs virtual address space subset filter [vstart..vend) */
        __le64 vstart;
        __le64 vend;

        /*
         * profile to convert to, single is denoted by
         * BTRFS_AVAIL_ALLOC_BIT_SINGLE
         */
        __le64 target;

        /* BTRFS_BALANCE_ARGS_* */
        __le64 flags;

        /*
         * BTRFS_BALANCE_ARGS_LIMIT with value 'limit'
         * BTRFS_BALANCE_ARGS_LIMIT_RANGE - the extend version can use minimum
         * and maximum
         */
        union {
                __le64 limit;
                struct {
                        __le32 limit_min;
                        __le32 limit_max;
                };
        };

        /*
         * Process chunks that cross stripes_min..stripes_max devices,
         * BTRFS_BALANCE_ARGS_STRIPES_RANGE
         */
        __le32 stripes_min;
        __le32 stripes_max;

        __le64 unused[6];
} __attribute__ ((__packed__));

/*
 * store balance parameters to disk so that balance can be properly
 * resumed after crash or unmount
 */
struct btrfs_balance_item {
        /* BTRFS_BALANCE_* */
        __le64 flags;

        struct btrfs_disk_balance_args data;
        struct btrfs_disk_balance_args meta;
        struct btrfs_disk_balance_args sys;

        __le64 unused[4];
} __attribute__ ((__packed__));

enum {
        BTRFS_FILE_EXTENT_INLINE   = 0,
        BTRFS_FILE_EXTENT_REG      = 1,
        BTRFS_FILE_EXTENT_PREALLOC = 2,
        BTRFS_NR_FILE_EXTENT_TYPES = 3,
};

struct btrfs_file_extent_item {
        /*
         * transaction id that created this extent
         */
        __le64 generation;
        /*
         * max number of bytes to hold this extent in ram
         * when we split a compressed extent we can't know how big
         * each of the resulting pieces will be.  So, this is
         * an upper limit on the size of the extent in ram instead of
         * an exact limit.
         */
        __le64 ram_bytes;

        /*
         * 32 bits for the various ways we might encode the data,
         * including compression and encryption.  If any of these
         * are set to something a given disk format doesn't understand
         * it is treated like an incompat flag for reading and writing,
         * but not for stat.
         */
        __u8 compression;
        __u8 encryption;
        __le16 other_encoding; /* spare for later use */

        /* are we inline data or a real extent? */
        __u8 type;

        /*
         * disk space consumed by the extent, checksum blocks are included
         * in these numbers
         *
         * At this offset in the structure, the inline extent data start.
         */
        __le64 disk_bytenr;
        __le64 disk_num_bytes;
        /*
         * the logical offset in file blocks (no csums)
         * this extent record is for.  This allows a file extent to point
         * into the middle of an existing extent on disk, sharing it
         * between two snapshots (useful if some bytes in the middle of the
         * extent have changed
         */
        __le64 offset;
        /*
         * the logical number of file blocks (no csums included).  This
         * always reflects the size uncompressed and without encoding.
         */
        __le64 num_bytes;

} __attribute__ ((__packed__));

struct btrfs_csum_item {
        __u8 csum;
} __attribute__ ((__packed__));

struct btrfs_dev_stats_item {
        /*
         * grow this item struct at the end for future enhancements and keep
         * the existing values unchanged
         */
        __le64 values[BTRFS_DEV_STAT_VALUES_MAX];
} __attribute__ ((__packed__));

#define BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_ALWAYS        0
#define BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID        1

struct btrfs_dev_replace_item {
        /*
         * grow this item struct at the end for future enhancements and keep
         * the existing values unchanged
         */
        __le64 src_devid;
        __le64 cursor_left;
        __le64 cursor_right;
        __le64 cont_reading_from_srcdev_mode;

        __le64 replace_state;
        __le64 time_started;
        __le64 time_stopped;
        __le64 num_write_errors;
        __le64 num_uncorrectable_read_errors;
} __attribute__ ((__packed__));

/* different types of block groups (and chunks) */
#define BTRFS_BLOCK_GROUP_DATA                (1ULL << 0)
#define BTRFS_BLOCK_GROUP_SYSTEM        (1ULL << 1)
#define BTRFS_BLOCK_GROUP_METADATA        (1ULL << 2)
#define BTRFS_BLOCK_GROUP_RAID0                (1ULL << 3)
#define BTRFS_BLOCK_GROUP_RAID1                (1ULL << 4)
#define BTRFS_BLOCK_GROUP_DUP                (1ULL << 5)
#define BTRFS_BLOCK_GROUP_RAID10        (1ULL << 6)
#define BTRFS_BLOCK_GROUP_RAID5         (1ULL << 7)
#define BTRFS_BLOCK_GROUP_RAID6         (1ULL << 8)
#define BTRFS_BLOCK_GROUP_RAID1C3       (1ULL << 9)
#define BTRFS_BLOCK_GROUP_RAID1C4       (1ULL << 10)
#define BTRFS_BLOCK_GROUP_RESERVED        (BTRFS_AVAIL_ALLOC_BIT_SINGLE | \
                                         BTRFS_SPACE_INFO_GLOBAL_RSV)

#define BTRFS_BLOCK_GROUP_TYPE_MASK        (BTRFS_BLOCK_GROUP_DATA |    \
                                         BTRFS_BLOCK_GROUP_SYSTEM |  \
                                         BTRFS_BLOCK_GROUP_METADATA)

#define BTRFS_BLOCK_GROUP_PROFILE_MASK        (BTRFS_BLOCK_GROUP_RAID0 |   \
                                         BTRFS_BLOCK_GROUP_RAID1 |   \
                                         BTRFS_BLOCK_GROUP_RAID1C3 | \
                                         BTRFS_BLOCK_GROUP_RAID1C4 | \
                                         BTRFS_BLOCK_GROUP_RAID5 |   \
                                         BTRFS_BLOCK_GROUP_RAID6 |   \
                                         BTRFS_BLOCK_GROUP_DUP |     \
                                         BTRFS_BLOCK_GROUP_RAID10)
#define BTRFS_BLOCK_GROUP_RAID56_MASK        (BTRFS_BLOCK_GROUP_RAID5 |   \
                                         BTRFS_BLOCK_GROUP_RAID6)

#define BTRFS_BLOCK_GROUP_RAID1_MASK        (BTRFS_BLOCK_GROUP_RAID1 |   \
                                         BTRFS_BLOCK_GROUP_RAID1C3 | \
                                         BTRFS_BLOCK_GROUP_RAID1C4)

/*
 * We need a bit for restriper to be able to tell when chunks of type
 * SINGLE are available.  This "extended" profile format is used in
 * fs_info->avail_*_alloc_bits (in-memory) and balance item fields
 * (on-disk).  The corresponding on-disk bit in chunk.type is reserved
 * to avoid remappings between two formats in future.
 */
#define BTRFS_AVAIL_ALLOC_BIT_SINGLE        (1ULL << 48)

/*
 * A fake block group type that is used to communicate global block reserve
 * size to userspace via the SPACE_INFO ioctl.
 */
#define BTRFS_SPACE_INFO_GLOBAL_RSV        (1ULL << 49)

#define BTRFS_EXTENDED_PROFILE_MASK        (BTRFS_BLOCK_GROUP_PROFILE_MASK | \
                                         BTRFS_AVAIL_ALLOC_BIT_SINGLE)

static inline __u64 chunk_to_extended(__u64 flags)
{
        if ((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0)
                flags |= BTRFS_AVAIL_ALLOC_BIT_SINGLE;

        return flags;
}
static inline __u64 extended_to_chunk(__u64 flags)
{
        return flags & ~BTRFS_AVAIL_ALLOC_BIT_SINGLE;
}

struct btrfs_block_group_item {
        __le64 used;
        __le64 chunk_objectid;
        __le64 flags;
} __attribute__ ((__packed__));

struct btrfs_free_space_info {
        __le32 extent_count;
        __le32 flags;
} __attribute__ ((__packed__));

#define BTRFS_FREE_SPACE_USING_BITMAPS (1ULL << 0)

#define BTRFS_QGROUP_LEVEL_SHIFT                48
static inline __u16 btrfs_qgroup_level(__u64 qgroupid)
{
        return (__u16)(qgroupid >> BTRFS_QGROUP_LEVEL_SHIFT);
}

/*
 * is subvolume quota turned on?
 */
#define BTRFS_QGROUP_STATUS_FLAG_ON                (1ULL << 0)
/*
 * RESCAN is set during the initialization phase
 */
#define BTRFS_QGROUP_STATUS_FLAG_RESCAN                (1ULL << 1)
/*
 * Some qgroup entries are known to be out of date,
 * either because the configuration has changed in a way that
 * makes a rescan necessary, or because the fs has been mounted
 * with a non-qgroup-aware version.
 * Turning qouta off and on again makes it inconsistent, too.
 */
#define BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT        (1ULL << 2)

/*
 * Whether or not this filesystem is using simple quotas.  Not exactly the
 * incompat bit, because we support using simple quotas, disabling it, then
 * going back to full qgroup quotas.
 */
#define BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE        (1ULL << 3)

#define BTRFS_QGROUP_STATUS_FLAGS_MASK        (BTRFS_QGROUP_STATUS_FLAG_ON |                \
                                         BTRFS_QGROUP_STATUS_FLAG_RESCAN |        \
                                         BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT | \
                                         BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE)

#define BTRFS_QGROUP_STATUS_VERSION        1

struct btrfs_qgroup_status_item {
        __le64 version;
        /*
         * the generation is updated during every commit. As older
         * versions of btrfs are not aware of qgroups, it will be
         * possible to detect inconsistencies by checking the
         * generation on mount time
         */
        __le64 generation;

        /* flag definitions see above */
        __le64 flags;

        /*
         * only used during scanning to record the progress
         * of the scan. It contains a logical address
         */
        __le64 rescan;

        /*
         * The generation when quotas were last enabled. Used by simple quotas to
         * avoid decrementing when freeing an extent that was written before
         * enable.
         *
         * Set only if flags contain BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE.
         */
        __le64 enable_gen;
} __attribute__ ((__packed__));

struct btrfs_qgroup_info_item {
        __le64 generation;
        __le64 rfer;
        __le64 rfer_cmpr;
        __le64 excl;
        __le64 excl_cmpr;
} __attribute__ ((__packed__));

struct btrfs_qgroup_limit_item {
        /*
         * only updated when any of the other values change
         */
        __le64 flags;
        __le64 max_rfer;
        __le64 max_excl;
        __le64 rsv_rfer;
        __le64 rsv_excl;
} __attribute__ ((__packed__));

struct btrfs_verity_descriptor_item {
        /* Size of the verity descriptor in bytes */
        __le64 size;
        /*
         * When we implement support for fscrypt, we will need to encrypt the
         * Merkle tree for encrypted verity files. These 128 bits are for the
         * eventual storage of an fscrypt initialization vector.
         */
        __le64 reserved[2];
        __u8 encryption;
} __attribute__ ((__packed__));

#endif /* _BTRFS_CTREE_H_ */







































































































































































































































































































































































































































































 4271 







































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399