Total coverage: 226540 (13%)of 1850944 
























































    2 










    2 














































































    1 










    1 















































    2 










    2 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
// SPDX-License-Identifier: GPL-2.0-only

#include <linux/phy.h>
#include <linux/ethtool_netlink.h>

#include "netlink.h"
#include "common.h"

struct plca_req_info {
        struct ethnl_req_info                base;
};

struct plca_reply_data {
        struct ethnl_reply_data                base;
        struct phy_plca_cfg                plca_cfg;
        struct phy_plca_status                plca_st;
};

// Helpers ------------------------------------------------------------------ //

#define PLCA_REPDATA(__reply_base) \
        container_of(__reply_base, struct plca_reply_data, base)

// PLCA get configuration message ------------------------------------------- //

const struct nla_policy ethnl_plca_get_cfg_policy[] = {
        [ETHTOOL_A_PLCA_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy_phy),
};

static void plca_update_sint(int *dst, struct nlattr **tb, u32 attrid,
                             bool *mod)
{
        const struct nlattr *attr = tb[attrid];

        if (!attr ||
            WARN_ON_ONCE(attrid >= ARRAY_SIZE(ethnl_plca_set_cfg_policy)))
                return;

        switch (ethnl_plca_set_cfg_policy[attrid].type) {
        case NLA_U8:
                *dst = nla_get_u8(attr);
                break;
        case NLA_U32:
                *dst = nla_get_u32(attr);
                break;
        default:
                WARN_ON_ONCE(1);
        }

        *mod = true;
}

static int plca_get_cfg_prepare_data(const struct ethnl_req_info *req_base,
                                     struct ethnl_reply_data *reply_base,
                                     const struct genl_info *info)
{
        struct plca_reply_data *data = PLCA_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        const struct ethtool_phy_ops *ops;
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        int ret;

        phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_PLCA_HEADER,
                                      info->extack);
        // check that the PHY device is available and connected
        if (IS_ERR_OR_NULL(phydev)) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        // note: rtnl_lock is held already by ethnl_default_doit
        ops = ethtool_phy_ops;
        if (!ops || !ops->get_plca_cfg) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                goto out;

        memset(&data->plca_cfg, 0xff,
               sizeof_field(struct plca_reply_data, plca_cfg));

        ret = ops->get_plca_cfg(phydev, &data->plca_cfg);
        ethnl_ops_complete(dev);

out:
        return ret;
}

static int plca_get_cfg_reply_size(const struct ethnl_req_info *req_base,
                                   const struct ethnl_reply_data *reply_base)
{
        return nla_total_size(sizeof(u16)) +        /* _VERSION */
               nla_total_size(sizeof(u8)) +        /* _ENABLED */
               nla_total_size(sizeof(u32)) +        /* _NODE_CNT */
               nla_total_size(sizeof(u32)) +        /* _NODE_ID */
               nla_total_size(sizeof(u32)) +        /* _TO_TIMER */
               nla_total_size(sizeof(u32)) +        /* _BURST_COUNT */
               nla_total_size(sizeof(u32));        /* _BURST_TIMER */
}

static int plca_get_cfg_fill_reply(struct sk_buff *skb,
                                   const struct ethnl_req_info *req_base,
                                   const struct ethnl_reply_data *reply_base)
{
        const struct plca_reply_data *data = PLCA_REPDATA(reply_base);
        const struct phy_plca_cfg *plca = &data->plca_cfg;

        if ((plca->version >= 0 &&
             nla_put_u16(skb, ETHTOOL_A_PLCA_VERSION, plca->version)) ||
            (plca->enabled >= 0 &&
             nla_put_u8(skb, ETHTOOL_A_PLCA_ENABLED, !!plca->enabled)) ||
            (plca->node_id >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_NODE_ID, plca->node_id)) ||
            (plca->node_cnt >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_NODE_CNT, plca->node_cnt)) ||
            (plca->to_tmr >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_TO_TMR, plca->to_tmr)) ||
            (plca->burst_cnt >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_BURST_CNT, plca->burst_cnt)) ||
            (plca->burst_tmr >= 0 &&
             nla_put_u32(skb, ETHTOOL_A_PLCA_BURST_TMR, plca->burst_tmr)))
                return -EMSGSIZE;

        return 0;
};

// PLCA set configuration message ------------------------------------------- //

const struct nla_policy ethnl_plca_set_cfg_policy[] = {
        [ETHTOOL_A_PLCA_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy_phy),
        [ETHTOOL_A_PLCA_ENABLED]        = NLA_POLICY_MAX(NLA_U8, 1),
        [ETHTOOL_A_PLCA_NODE_ID]        = NLA_POLICY_MAX(NLA_U32, 255),
        [ETHTOOL_A_PLCA_NODE_CNT]        = NLA_POLICY_RANGE(NLA_U32, 1, 255),
        [ETHTOOL_A_PLCA_TO_TMR]                = NLA_POLICY_MAX(NLA_U32, 255),
        [ETHTOOL_A_PLCA_BURST_CNT]        = NLA_POLICY_MAX(NLA_U32, 255),
        [ETHTOOL_A_PLCA_BURST_TMR]        = NLA_POLICY_MAX(NLA_U32, 255),
};

static int
ethnl_set_plca(struct ethnl_req_info *req_info, struct genl_info *info)
{
        const struct ethtool_phy_ops *ops;
        struct nlattr **tb = info->attrs;
        struct phy_plca_cfg plca_cfg;
        struct phy_device *phydev;
        bool mod = false;
        int ret;

        phydev = ethnl_req_get_phydev(req_info, tb, ETHTOOL_A_PLCA_HEADER,
                                      info->extack);
        // check that the PHY device is available and connected
        if (IS_ERR_OR_NULL(phydev))
                return -EOPNOTSUPP;

        ops = ethtool_phy_ops;
        if (!ops || !ops->set_plca_cfg)
                return -EOPNOTSUPP;

        memset(&plca_cfg, 0xff, sizeof(plca_cfg));
        plca_update_sint(&plca_cfg.enabled, tb, ETHTOOL_A_PLCA_ENABLED, &mod);
        plca_update_sint(&plca_cfg.node_id, tb, ETHTOOL_A_PLCA_NODE_ID, &mod);
        plca_update_sint(&plca_cfg.node_cnt, tb, ETHTOOL_A_PLCA_NODE_CNT, &mod);
        plca_update_sint(&plca_cfg.to_tmr, tb, ETHTOOL_A_PLCA_TO_TMR, &mod);
        plca_update_sint(&plca_cfg.burst_cnt, tb, ETHTOOL_A_PLCA_BURST_CNT,
                         &mod);
        plca_update_sint(&plca_cfg.burst_tmr, tb, ETHTOOL_A_PLCA_BURST_TMR,
                         &mod);
        if (!mod)
                return 0;

        ret = ops->set_plca_cfg(phydev, &plca_cfg, info->extack);
        return ret < 0 ? ret : 1;
}

const struct ethnl_request_ops ethnl_plca_cfg_request_ops = {
        .request_cmd                = ETHTOOL_MSG_PLCA_GET_CFG,
        .reply_cmd                = ETHTOOL_MSG_PLCA_GET_CFG_REPLY,
        .hdr_attr                = ETHTOOL_A_PLCA_HEADER,
        .req_info_size                = sizeof(struct plca_req_info),
        .reply_data_size        = sizeof(struct plca_reply_data),

        .prepare_data                = plca_get_cfg_prepare_data,
        .reply_size                = plca_get_cfg_reply_size,
        .fill_reply                = plca_get_cfg_fill_reply,

        .set                        = ethnl_set_plca,
        .set_ntf_cmd                = ETHTOOL_MSG_PLCA_NTF,
};

// PLCA get status message -------------------------------------------------- //

const struct nla_policy ethnl_plca_get_status_policy[] = {
        [ETHTOOL_A_PLCA_HEADER]                =
                NLA_POLICY_NESTED(ethnl_header_policy_phy),
};

static int plca_get_status_prepare_data(const struct ethnl_req_info *req_base,
                                        struct ethnl_reply_data *reply_base,
                                        const struct genl_info *info)
{
        struct plca_reply_data *data = PLCA_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        const struct ethtool_phy_ops *ops;
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        int ret;

        phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_PLCA_HEADER,
                                      info->extack);
        // check that the PHY device is available and connected
        if (IS_ERR_OR_NULL(phydev)) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        // note: rtnl_lock is held already by ethnl_default_doit
        ops = ethtool_phy_ops;
        if (!ops || !ops->get_plca_status) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                goto out;

        memset(&data->plca_st, 0xff,
               sizeof_field(struct plca_reply_data, plca_st));

        ret = ops->get_plca_status(phydev, &data->plca_st);
        ethnl_ops_complete(dev);
out:
        return ret;
}

static int plca_get_status_reply_size(const struct ethnl_req_info *req_base,
                                      const struct ethnl_reply_data *reply_base)
{
        return nla_total_size(sizeof(u8));        /* _STATUS */
}

static int plca_get_status_fill_reply(struct sk_buff *skb,
                                      const struct ethnl_req_info *req_base,
                                      const struct ethnl_reply_data *reply_base)
{
        const struct plca_reply_data *data = PLCA_REPDATA(reply_base);
        const u8 status = data->plca_st.pst;

        if (nla_put_u8(skb, ETHTOOL_A_PLCA_STATUS, !!status))
                return -EMSGSIZE;

        return 0;
};

const struct ethnl_request_ops ethnl_plca_status_request_ops = {
        .request_cmd                = ETHTOOL_MSG_PLCA_GET_STATUS,
        .reply_cmd                = ETHTOOL_MSG_PLCA_GET_STATUS_REPLY,
        .hdr_attr                = ETHTOOL_A_PLCA_HEADER,
        .req_info_size                = sizeof(struct plca_req_info),
        .reply_data_size        = sizeof(struct plca_reply_data),

        .prepare_data                = plca_get_status_prepare_data,
        .reply_size                = plca_get_status_reply_size,
        .fill_reply                = plca_get_status_fill_reply,
};




























  450 







































  441 











































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM snd_pcm

#if !defined(_PCM_PARAMS_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
#define _PCM_PARAMS_TRACE_H

#include <linux/tracepoint.h>

#define HW_PARAM_ENTRY(param) {SNDRV_PCM_HW_PARAM_##param, #param}
#define hw_param_labels                        \
        HW_PARAM_ENTRY(ACCESS),                \
        HW_PARAM_ENTRY(FORMAT),                \
        HW_PARAM_ENTRY(SUBFORMAT),        \
        HW_PARAM_ENTRY(SAMPLE_BITS),        \
        HW_PARAM_ENTRY(FRAME_BITS),        \
        HW_PARAM_ENTRY(CHANNELS),        \
        HW_PARAM_ENTRY(RATE),                \
        HW_PARAM_ENTRY(PERIOD_TIME),        \
        HW_PARAM_ENTRY(PERIOD_SIZE),        \
        HW_PARAM_ENTRY(PERIOD_BYTES),        \
        HW_PARAM_ENTRY(PERIODS),        \
        HW_PARAM_ENTRY(BUFFER_TIME),        \
        HW_PARAM_ENTRY(BUFFER_SIZE),        \
        HW_PARAM_ENTRY(BUFFER_BYTES),        \
        HW_PARAM_ENTRY(TICK_TIME)

TRACE_EVENT(hw_mask_param,
        TP_PROTO(struct snd_pcm_substream *substream, snd_pcm_hw_param_t type, int index, const struct snd_mask *prev, const struct snd_mask *curr),
        TP_ARGS(substream, type, index, prev, curr),
        TP_STRUCT__entry(
                __field(int, card)
                __field(int, device)
                __field(int, subdevice)
                __field(int, direction)
                __field(snd_pcm_hw_param_t, type)
                __field(int, index)
                __field(int, total)
                __array(__u32, prev_bits, 8)
                __array(__u32, curr_bits, 8)
        ),
        TP_fast_assign(
                __entry->card = substream->pcm->card->number;
                __entry->device = substream->pcm->device;
                __entry->subdevice = substream->number;
                __entry->direction = substream->stream;
                __entry->type = type;
                __entry->index = index;
                __entry->total = substream->runtime->hw_constraints.rules_num;
                memcpy(__entry->prev_bits, prev->bits, sizeof(__u32) * 8);
                memcpy(__entry->curr_bits, curr->bits, sizeof(__u32) * 8);
        ),
        TP_printk("pcmC%dD%d%s:%d %03d/%03d %s %08x%08x%08x%08x %08x%08x%08x%08x",
                  __entry->card,
                  __entry->device,
                  __entry->direction ? "c" : "p",
                  __entry->subdevice,
                  __entry->index,
                  __entry->total,
                  __print_symbolic(__entry->type, hw_param_labels),
                  __entry->prev_bits[3], __entry->prev_bits[2],
                  __entry->prev_bits[1], __entry->prev_bits[0],
                  __entry->curr_bits[3], __entry->curr_bits[2],
                  __entry->curr_bits[1], __entry->curr_bits[0]
        )
);

TRACE_EVENT(hw_interval_param,
        TP_PROTO(struct snd_pcm_substream *substream, snd_pcm_hw_param_t type, int index, const struct snd_interval *prev, const struct snd_interval *curr),
        TP_ARGS(substream, type, index, prev, curr),
        TP_STRUCT__entry(
                __field(int, card)
                __field(int, device)
                __field(int, subdevice)
                __field(int, direction)
                __field(snd_pcm_hw_param_t, type)
                __field(int, index)
                __field(int, total)
                __field(unsigned int, prev_min)
                __field(unsigned int, prev_max)
                __field(unsigned int, prev_openmin)
                __field(unsigned int, prev_openmax)
                __field(unsigned int, prev_integer)
                __field(unsigned int, prev_empty)
                __field(unsigned int, curr_min)
                __field(unsigned int, curr_max)
                __field(unsigned int, curr_openmin)
                __field(unsigned int, curr_openmax)
                __field(unsigned int, curr_integer)
                __field(unsigned int, curr_empty)
        ),
        TP_fast_assign(
                __entry->card = substream->pcm->card->number;
                __entry->device = substream->pcm->device;
                __entry->subdevice = substream->number;
                __entry->direction = substream->stream;
                __entry->type = type;
                __entry->index = index;
                __entry->total = substream->runtime->hw_constraints.rules_num;
                __entry->prev_min = prev->min;
                __entry->prev_max = prev->max;
                __entry->prev_openmin = prev->openmin;
                __entry->prev_openmax = prev->openmax;
                __entry->prev_integer = prev->integer;
                __entry->prev_empty = prev->empty;
                __entry->curr_min = curr->min;
                __entry->curr_max = curr->max;
                __entry->curr_openmin = curr->openmin;
                __entry->curr_openmax = curr->openmax;
                __entry->curr_integer = curr->integer;
                __entry->curr_empty = curr->empty;
        ),
        TP_printk("pcmC%dD%d%s:%d %03d/%03d %s %d %d %s%u %u%s %d %d %s%u %u%s",
                  __entry->card,
                  __entry->device,
                  __entry->direction ? "c" : "p",
                  __entry->subdevice,
                  __entry->index,
                  __entry->total,
                  __print_symbolic(__entry->type, hw_param_labels),
                  __entry->prev_empty,
                  __entry->prev_integer,
                  __entry->prev_openmin ? "(" : "[",
                  __entry->prev_min,
                  __entry->prev_max,
                  __entry->prev_openmax ? ")" : "]",
                  __entry->curr_empty,
                  __entry->curr_integer,
                  __entry->curr_openmin ? "(" : "[",
                  __entry->curr_min,
                  __entry->curr_max,
                  __entry->curr_openmax ? ")" : "]"
        )
);

#endif /* _PCM_PARAMS_TRACE_H */

/* This part must be outside protection */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE pcm_param_trace
#include <trace/define_trace.h>
















































































































   35 

    1 




    8 
   27 
   27 
    8 




    8 









    8 

















































    3 













    3 





    2 






    3 











   15 





    1 






    1 



    9 


   10 






    2 


    3 



    1 

    8 



    9 


    1 

    1 









    9 
























    1 

















    1 













    1 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
/*
 * net/tipc/net.c: TIPC network routing code
 *
 * Copyright (c) 1995-2006, 2014, Ericsson AB
 * Copyright (c) 2005, 2010-2011, Wind River Systems
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "core.h"
#include "net.h"
#include "name_distr.h"
#include "subscr.h"
#include "socket.h"
#include "node.h"
#include "bcast.h"
#include "link.h"
#include "netlink.h"
#include "monitor.h"

/*
 * The TIPC locking policy is designed to ensure a very fine locking
 * granularity, permitting complete parallel access to individual
 * port and node/link instances. The code consists of four major
 * locking domains, each protected with their own disjunct set of locks.
 *
 * 1: The bearer level.
 *    RTNL lock is used to serialize the process of configuring bearer
 *    on update side, and RCU lock is applied on read side to make
 *    bearer instance valid on both paths of message transmission and
 *    reception.
 *
 * 2: The node and link level.
 *    All node instances are saved into two tipc_node_list and node_htable
 *    lists. The two lists are protected by node_list_lock on write side,
 *    and they are guarded with RCU lock on read side. Especially node
 *    instance is destroyed only when TIPC module is removed, and we can
 *    confirm that there has no any user who is accessing the node at the
 *    moment. Therefore, Except for iterating the two lists within RCU
 *    protection, it's no needed to hold RCU that we access node instance
 *    in other places.
 *
 *    In addition, all members in node structure including link instances
 *    are protected by node spin lock.
 *
 * 3: The transport level of the protocol.
 *    This consists of the structures port, (and its user level
 *    representations, such as user_port and tipc_sock), reference and
 *    tipc_user (port.c, reg.c, socket.c).
 *
 *    This layer has four different locks:
 *     - The tipc_port spin_lock. This is protecting each port instance
 *       from parallel data access and removal. Since we can not place
 *       this lock in the port itself, it has been placed in the
 *       corresponding reference table entry, which has the same life
 *       cycle as the module. This entry is difficult to access from
 *       outside the TIPC core, however, so a pointer to the lock has
 *       been added in the port instance, -to be used for unlocking
 *       only.
 *     - A read/write lock to protect the reference table itself (teg.c).
 *       (Nobody is using read-only access to this, so it can just as
 *       well be changed to a spin_lock)
 *     - A spin lock to protect the registry of kernel/driver users (reg.c)
 *     - A global spin_lock (tipc_port_lock), which only task is to ensure
 *       consistency where more than one port is involved in an operation,
 *       i.e., when a port is part of a linked list of ports.
 *       There are two such lists; 'port_list', which is used for management,
 *       and 'wait_list', which is used to queue ports during congestion.
 *
 *  4: The name table (name_table.c, name_distr.c, subscription.c)
 *     - There is one big read/write-lock (tipc_nametbl_lock) protecting the
 *       overall name table structure. Nothing must be added/removed to
 *       this structure without holding write access to it.
 *     - There is one local spin_lock per sub_sequence, which can be seen
 *       as a sub-domain to the tipc_nametbl_lock domain. It is used only
 *       for translation operations, and is needed because a translation
 *       steps the root of the 'publication' linked list between each lookup.
 *       This is always used within the scope of a tipc_nametbl_lock(read).
 *     - A local spin_lock protecting the queue of subscriber events.
*/

static void tipc_net_finalize(struct net *net, u32 addr);

int tipc_net_init(struct net *net, u8 *node_id, u32 addr)
{
        if (tipc_own_id(net)) {
                pr_info("Cannot configure node identity twice\n");
                return -1;
        }
        pr_info("Started in network mode\n");

        if (node_id)
                tipc_set_node_id(net, node_id);
        if (addr)
                tipc_net_finalize(net, addr);
        return 0;
}

static void tipc_net_finalize(struct net *net, u32 addr)
{
        struct tipc_net *tn = tipc_net(net);
        struct tipc_socket_addr sk = {0, addr};
        struct tipc_uaddr ua;

        tipc_uaddr(&ua, TIPC_SERVICE_RANGE, TIPC_CLUSTER_SCOPE,
                   TIPC_NODE_STATE, addr, addr);

        if (cmpxchg(&tn->node_addr, 0, addr))
                return;
        tipc_set_node_addr(net, addr);
        tipc_named_reinit(net);
        tipc_sk_reinit(net);
        tipc_mon_reinit_self(net);
        tipc_nametbl_publish(net, &ua, &sk, addr);
}

void tipc_net_finalize_work(struct work_struct *work)
{
        struct tipc_net *tn = container_of(work, struct tipc_net, work);

        tipc_net_finalize(tipc_link_net(tn->bcl), tn->trial_addr);
}

void tipc_net_stop(struct net *net)
{
        if (!tipc_own_id(net))
                return;

        rtnl_lock();
        tipc_bearer_stop(net);
        tipc_node_stop(net);
        rtnl_unlock();

        pr_info("Left network mode\n");
}

static int __tipc_nl_add_net(struct net *net, struct tipc_nl_msg *msg)
{
        struct tipc_net *tn = net_generic(net, tipc_net_id);
        u64 *w0 = (u64 *)&tn->node_id[0];
        u64 *w1 = (u64 *)&tn->node_id[8];
        struct nlattr *attrs;
        void *hdr;

        hdr = genlmsg_put(msg->skb, msg->portid, msg->seq, &tipc_genl_family,
                          NLM_F_MULTI, TIPC_NL_NET_GET);
        if (!hdr)
                return -EMSGSIZE;

        attrs = nla_nest_start_noflag(msg->skb, TIPC_NLA_NET);
        if (!attrs)
                goto msg_full;

        if (nla_put_u32(msg->skb, TIPC_NLA_NET_ID, tn->net_id))
                goto attr_msg_full;
        if (nla_put_u64_64bit(msg->skb, TIPC_NLA_NET_NODEID, *w0, 0))
                goto attr_msg_full;
        if (nla_put_u64_64bit(msg->skb, TIPC_NLA_NET_NODEID_W1, *w1, 0))
                goto attr_msg_full;
        nla_nest_end(msg->skb, attrs);
        genlmsg_end(msg->skb, hdr);

        return 0;

attr_msg_full:
        nla_nest_cancel(msg->skb, attrs);
msg_full:
        genlmsg_cancel(msg->skb, hdr);

        return -EMSGSIZE;
}

int tipc_nl_net_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct net *net = sock_net(skb->sk);
        int err;
        int done = cb->args[0];
        struct tipc_nl_msg msg;

        if (done)
                return 0;

        msg.skb = skb;
        msg.portid = NETLINK_CB(cb->skb).portid;
        msg.seq = cb->nlh->nlmsg_seq;

        err = __tipc_nl_add_net(net, &msg);
        if (err)
                goto out;

        done = 1;
out:
        cb->args[0] = done;

        return skb->len;
}

int __tipc_nl_net_set(struct sk_buff *skb, struct genl_info *info)
{
        struct nlattr *attrs[TIPC_NLA_NET_MAX + 1];
        struct net *net = sock_net(skb->sk);
        struct tipc_net *tn = tipc_net(net);
        int err;

        if (!info->attrs[TIPC_NLA_NET])
                return -EINVAL;

        err = nla_parse_nested_deprecated(attrs, TIPC_NLA_NET_MAX,
                                          info->attrs[TIPC_NLA_NET],
                                          tipc_nl_net_policy, info->extack);

        if (err)
                return err;

        /* Can't change net id once TIPC has joined a network */
        if (tipc_own_addr(net))
                return -EPERM;

        if (attrs[TIPC_NLA_NET_ID]) {
                u32 val;

                val = nla_get_u32(attrs[TIPC_NLA_NET_ID]);
                if (val < 1 || val > 9999)
                        return -EINVAL;

                tn->net_id = val;
        }

        if (attrs[TIPC_NLA_NET_ADDR]) {
                u32 addr;

                addr = nla_get_u32(attrs[TIPC_NLA_NET_ADDR]);
                if (!addr)
                        return -EINVAL;
                tn->legacy_addr_format = true;
                tipc_net_init(net, NULL, addr);
        }

        if (attrs[TIPC_NLA_NET_NODEID]) {
                u8 node_id[NODE_ID_LEN];
                u64 *w0 = (u64 *)&node_id[0];
                u64 *w1 = (u64 *)&node_id[8];

                if (!attrs[TIPC_NLA_NET_NODEID_W1])
                        return -EINVAL;
                *w0 = nla_get_u64(attrs[TIPC_NLA_NET_NODEID]);
                *w1 = nla_get_u64(attrs[TIPC_NLA_NET_NODEID_W1]);
                tipc_net_init(net, node_id, 0);
        }
        return 0;
}

int tipc_nl_net_set(struct sk_buff *skb, struct genl_info *info)
{
        int err;

        rtnl_lock();
        err = __tipc_nl_net_set(skb, info);
        rtnl_unlock();

        return err;
}

static int __tipc_nl_addr_legacy_get(struct net *net, struct tipc_nl_msg *msg)
{
        struct tipc_net *tn = tipc_net(net);
        struct nlattr *attrs;
        void *hdr;

        hdr = genlmsg_put(msg->skb, msg->portid, msg->seq, &tipc_genl_family,
                          0, TIPC_NL_ADDR_LEGACY_GET);
        if (!hdr)
                return -EMSGSIZE;

        attrs = nla_nest_start(msg->skb, TIPC_NLA_NET);
        if (!attrs)
                goto msg_full;

        if (tn->legacy_addr_format)
                if (nla_put_flag(msg->skb, TIPC_NLA_NET_ADDR_LEGACY))
                        goto attr_msg_full;

        nla_nest_end(msg->skb, attrs);
        genlmsg_end(msg->skb, hdr);

        return 0;

attr_msg_full:
        nla_nest_cancel(msg->skb, attrs);
msg_full:
        genlmsg_cancel(msg->skb, hdr);

        return -EMSGSIZE;
}

int tipc_nl_net_addr_legacy_get(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = sock_net(skb->sk);
        struct tipc_nl_msg msg;
        struct sk_buff *rep;
        int err;

        rep = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!rep)
                return -ENOMEM;

        msg.skb = rep;
        msg.portid = info->snd_portid;
        msg.seq = info->snd_seq;

        err = __tipc_nl_addr_legacy_get(net, &msg);
        if (err) {
                nlmsg_free(msg.skb);
                return err;
        }

        return genlmsg_reply(msg.skb, info);
}






























































































































































































































































































































  200 


















































  140 





















































































































  180 
  192 
































































































































 6349 






















  129 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SWAP_H
#define _LINUX_SWAP_H

#include <linux/spinlock.h>
#include <linux/linkage.h>
#include <linux/mmzone.h>
#include <linux/list.h>
#include <linux/memcontrol.h>
#include <linux/sched.h>
#include <linux/node.h>
#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/atomic.h>
#include <linux/page-flags.h>
#include <uapi/linux/mempolicy.h>
#include <asm/page.h>

struct notifier_block;

struct bio;

struct pagevec;

#define SWAP_FLAG_PREFER        0x8000        /* set if swap priority specified */
#define SWAP_FLAG_PRIO_MASK        0x7fff
#define SWAP_FLAG_DISCARD        0x10000 /* enable discard for swap */
#define SWAP_FLAG_DISCARD_ONCE        0x20000 /* discard swap area at swapon-time */
#define SWAP_FLAG_DISCARD_PAGES 0x40000 /* discard page-clusters after use */

#define SWAP_FLAGS_VALID        (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \
                                 SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \
                                 SWAP_FLAG_DISCARD_PAGES)
#define SWAP_BATCH 64

static inline int current_is_kswapd(void)
{
        return current->flags & PF_KSWAPD;
}

/*
 * MAX_SWAPFILES defines the maximum number of swaptypes: things which can
 * be swapped to.  The swap type and the offset into that swap type are
 * encoded into pte's and into pgoff_t's in the swapcache.  Using five bits
 * for the type means that the maximum number of swapcache pages is 27 bits
 * on 32-bit-pgoff_t architectures.  And that assumes that the architecture packs
 * the type/offset into the pte as 5/27 as well.
 */
#define MAX_SWAPFILES_SHIFT        5

/*
 * Use some of the swap files numbers for other purposes. This
 * is a convenient way to hook into the VM to trigger special
 * actions on faults.
 */

/*
 * PTE markers are used to persist information onto PTEs that otherwise
 * should be a none pte.  As its name "PTE" hints, it should only be
 * applied to the leaves of pgtables.
 */
#define SWP_PTE_MARKER_NUM 1
#define SWP_PTE_MARKER     (MAX_SWAPFILES + SWP_HWPOISON_NUM + \
                            SWP_MIGRATION_NUM + SWP_DEVICE_NUM)

/*
 * Unaddressable device memory support. See include/linux/hmm.h and
 * Documentation/mm/hmm.rst. Short description is we need struct pages for
 * device memory that is unaddressable (inaccessible) by CPU, so that we can
 * migrate part of a process memory to device memory.
 *
 * When a page is migrated from CPU to device, we set the CPU page table entry
 * to a special SWP_DEVICE_{READ|WRITE} entry.
 *
 * When a page is mapped by the device for exclusive access we set the CPU page
 * table entries to a special SWP_DEVICE_EXCLUSIVE entry.
 */
#ifdef CONFIG_DEVICE_PRIVATE
#define SWP_DEVICE_NUM 3
#define SWP_DEVICE_WRITE (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM)
#define SWP_DEVICE_READ (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM+1)
#define SWP_DEVICE_EXCLUSIVE (MAX_SWAPFILES+SWP_HWPOISON_NUM+SWP_MIGRATION_NUM+2)
#else
#define SWP_DEVICE_NUM 0
#endif

/*
 * Page migration support.
 *
 * SWP_MIGRATION_READ_EXCLUSIVE is only applicable to anonymous pages and
 * indicates that the referenced (part of) an anonymous page is exclusive to
 * a single process. For SWP_MIGRATION_WRITE, that information is implicit:
 * (part of) an anonymous page that are mapped writable are exclusive to a
 * single process.
 */
#ifdef CONFIG_MIGRATION
#define SWP_MIGRATION_NUM 3
#define SWP_MIGRATION_READ (MAX_SWAPFILES + SWP_HWPOISON_NUM)
#define SWP_MIGRATION_READ_EXCLUSIVE (MAX_SWAPFILES + SWP_HWPOISON_NUM + 1)
#define SWP_MIGRATION_WRITE (MAX_SWAPFILES + SWP_HWPOISON_NUM + 2)
#else
#define SWP_MIGRATION_NUM 0
#endif

/*
 * Handling of hardware poisoned pages with memory corruption.
 */
#ifdef CONFIG_MEMORY_FAILURE
#define SWP_HWPOISON_NUM 1
#define SWP_HWPOISON                MAX_SWAPFILES
#else
#define SWP_HWPOISON_NUM 0
#endif

#define MAX_SWAPFILES \
        ((1 << MAX_SWAPFILES_SHIFT) - SWP_DEVICE_NUM - \
        SWP_MIGRATION_NUM - SWP_HWPOISON_NUM - \
        SWP_PTE_MARKER_NUM)

/*
 * Magic header for a swap area. The first part of the union is
 * what the swap magic looks like for the old (limited to 128MB)
 * swap area format, the second part of the union adds - in the
 * old reserved area - some extra information. Note that the first
 * kilobyte is reserved for boot loader or disk label stuff...
 *
 * Having the magic at the end of the PAGE_SIZE makes detecting swap
 * areas somewhat tricky on machines that support multiple page sizes.
 * For 2.5 we'll probably want to move the magic to just beyond the
 * bootbits...
 */
union swap_header {
        struct {
                char reserved[PAGE_SIZE - 10];
                char magic[10];                        /* SWAP-SPACE or SWAPSPACE2 */
        } magic;
        struct {
                char                bootbits[1024];        /* Space for disklabel etc. */
                __u32                version;
                __u32                last_page;
                __u32                nr_badpages;
                unsigned char        sws_uuid[16];
                unsigned char        sws_volume[16];
                __u32                padding[117];
                __u32                badpages[1];
        } info;
};

/*
 * current->reclaim_state points to one of these when a task is running
 * memory reclaim
 */
struct reclaim_state {
        /* pages reclaimed outside of LRU-based reclaim */
        unsigned long reclaimed;
#ifdef CONFIG_LRU_GEN
        /* per-thread mm walk data */
        struct lru_gen_mm_walk *mm_walk;
#endif
};

/*
 * mm_account_reclaimed_pages(): account reclaimed pages outside of LRU-based
 * reclaim
 * @pages: number of pages reclaimed
 *
 * If the current process is undergoing a reclaim operation, increment the
 * number of reclaimed pages by @pages.
 */
static inline void mm_account_reclaimed_pages(unsigned long pages)
{
        if (current->reclaim_state)
                current->reclaim_state->reclaimed += pages;
}

#ifdef __KERNEL__

struct address_space;
struct sysinfo;
struct writeback_control;
struct zone;

/*
 * A swap extent maps a range of a swapfile's PAGE_SIZE pages onto a range of
 * disk blocks.  A rbtree of swap extents maps the entire swapfile (Where the
 * term `swapfile' refers to either a blockdevice or an IS_REG file). Apart
 * from setup, they're handled identically.
 *
 * We always assume that blocks are of size PAGE_SIZE.
 */
struct swap_extent {
        struct rb_node rb_node;
        pgoff_t start_page;
        pgoff_t nr_pages;
        sector_t start_block;
};

/*
 * Max bad pages in the new format..
 */
#define MAX_SWAP_BADPAGES \
        ((offsetof(union swap_header, magic.magic) - \
          offsetof(union swap_header, info.badpages)) / sizeof(int))

enum {
        SWP_USED        = (1 << 0),        /* is slot in swap_info[] used? */
        SWP_WRITEOK        = (1 << 1),        /* ok to write to this swap?        */
        SWP_DISCARDABLE = (1 << 2),        /* blkdev support discard */
        SWP_DISCARDING        = (1 << 3),        /* now discarding a free cluster */
        SWP_SOLIDSTATE        = (1 << 4),        /* blkdev seeks are cheap */
        SWP_CONTINUED        = (1 << 5),        /* swap_map has count continuation */
        SWP_BLKDEV        = (1 << 6),        /* its a block device */
        SWP_ACTIVATED        = (1 << 7),        /* set after swap_activate success */
        SWP_FS_OPS        = (1 << 8),        /* swapfile operations go through fs */
        SWP_AREA_DISCARD = (1 << 9),        /* single-time swap area discards */
        SWP_PAGE_DISCARD = (1 << 10),        /* freed swap page-cluster discards */
        SWP_STABLE_WRITES = (1 << 11),        /* no overwrite PG_writeback pages */
        SWP_SYNCHRONOUS_IO = (1 << 12),        /* synchronous IO is efficient */
                                        /* add others here before... */
};

#define SWAP_CLUSTER_MAX 32UL
#define SWAP_CLUSTER_MAX_SKIPPED (SWAP_CLUSTER_MAX << 10)
#define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX

/* Bit flag in swap_map */
#define SWAP_HAS_CACHE        0x40        /* Flag page is cached, in first swap_map */
#define COUNT_CONTINUED        0x80        /* Flag swap_map continuation for full count */

/* Special value in first swap_map */
#define SWAP_MAP_MAX        0x3e        /* Max count */
#define SWAP_MAP_BAD        0x3f        /* Note page is bad */
#define SWAP_MAP_SHMEM        0xbf        /* Owned by shmem/tmpfs */

/* Special value in each swap_map continuation */
#define SWAP_CONT_MAX        0x7f        /* Max count */

/*
 * The first page in the swap file is the swap header, which is always marked
 * bad to prevent it from being allocated as an entry. This also prevents the
 * cluster to which it belongs being marked free. Therefore 0 is safe to use as
 * a sentinel to indicate an entry is not valid.
 */
#define SWAP_ENTRY_INVALID        0

#ifdef CONFIG_THP_SWAP
#define SWAP_NR_ORDERS                (PMD_ORDER + 1)
#else
#define SWAP_NR_ORDERS                1
#endif

/*
 * We keep using same cluster for rotational device so IO will be sequential.
 * The purpose is to optimize SWAP throughput on these device.
 */
struct swap_sequential_cluster {
        unsigned int next[SWAP_NR_ORDERS]; /* Likely next allocation offset */
};

/*
 * The in-memory structure used to track swap areas.
 */
struct swap_info_struct {
        struct percpu_ref users;        /* indicate and keep swap device valid. */
        unsigned long        flags;                /* SWP_USED etc: see above */
        signed short        prio;                /* swap priority of this type */
        struct plist_node list;                /* entry in swap_active_head */
        signed char        type;                /* strange name for an index */
        unsigned int        max;                /* extent of the swap_map */
        unsigned char *swap_map;        /* vmalloc'ed array of usage counts */
        unsigned long *zeromap;                /* kvmalloc'ed bitmap to track zero pages */
        struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
        struct list_head free_clusters; /* free clusters list */
        struct list_head full_clusters; /* full clusters list */
        struct list_head nonfull_clusters[SWAP_NR_ORDERS];
                                        /* list of cluster that contains at least one free slot */
        struct list_head frag_clusters[SWAP_NR_ORDERS];
                                        /* list of cluster that are fragmented or contented */
        unsigned int pages;                /* total of usable pages of swap */
        atomic_long_t inuse_pages;        /* number of those currently in use */
        struct swap_sequential_cluster *global_cluster; /* Use one global cluster for rotating device */
        spinlock_t global_cluster_lock;        /* Serialize usage of global cluster */
        struct rb_root swap_extent_root;/* root of the swap extent rbtree */
        struct block_device *bdev;        /* swap device or bdev of swap file */
        struct file *swap_file;                /* seldom referenced */
        struct completion comp;                /* seldom referenced */
        spinlock_t lock;                /*
                                         * protect map scan related fields like
                                         * swap_map, inuse_pages and all cluster
                                         * lists. other fields are only changed
                                         * at swapon/swapoff, so are protected
                                         * by swap_lock. changing flags need
                                         * hold this lock and swap_lock. If
                                         * both locks need hold, hold swap_lock
                                         * first.
                                         */
        spinlock_t cont_lock;                /*
                                         * protect swap count continuation page
                                         * list.
                                         */
        struct work_struct discard_work; /* discard worker */
        struct work_struct reclaim_work; /* reclaim worker */
        struct list_head discard_clusters; /* discard clusters list */
        struct plist_node avail_lists[]; /*
                                           * entries in swap_avail_heads, one
                                           * entry per node.
                                           * Must be last as the number of the
                                           * array is nr_node_ids, which is not
                                           * a fixed value so have to allocate
                                           * dynamically.
                                           * And it has to be an array so that
                                           * plist_for_each_* can work.
                                           */
};

static inline swp_entry_t page_swap_entry(struct page *page)
{
        struct folio *folio = page_folio(page);
        swp_entry_t entry = folio->swap;

        entry.val += folio_page_idx(folio, page);
        return entry;
}

/* linux/mm/workingset.c */
bool workingset_test_recent(void *shadow, bool file, bool *workingset,
                                bool flush);
void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg);
void workingset_refault(struct folio *folio, void *shadow);
void workingset_activation(struct folio *folio);

/* linux/mm/page_alloc.c */
extern unsigned long totalreserve_pages;

/* Definition of global_zone_page_state not available yet */
#define nr_free_pages() global_zone_page_state(NR_FREE_PAGES)


/* linux/mm/swap.c */
void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
                unsigned int nr_io, unsigned int nr_rotated)
                __releases(lruvec->lru_lock);
void lru_note_cost_refault(struct folio *);
void folio_add_lru(struct folio *);
void folio_add_lru_vma(struct folio *, struct vm_area_struct *);
void mark_page_accessed(struct page *);
void folio_mark_accessed(struct folio *);

static inline bool folio_may_be_lru_cached(struct folio *folio)
{
        /*
         * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting.
         * Holding small numbers of low-order mTHP folios in per-CPU LRU cache
         * will be sensible, but nobody has implemented and tested that yet.
         */
        return !folio_test_large(folio);
}

extern atomic_t lru_disable_count;

static inline bool lru_cache_disabled(void)
{
        return atomic_read(&lru_disable_count);
}

static inline void lru_cache_enable(void)
{
        atomic_dec(&lru_disable_count);
}

extern void lru_cache_disable(void);
extern void lru_add_drain(void);
extern void lru_add_drain_cpu(int cpu);
extern void lru_add_drain_cpu_zone(struct zone *zone);
extern void lru_add_drain_all(void);
void folio_deactivate(struct folio *folio);
void folio_mark_lazyfree(struct folio *folio);
extern void swap_setup(void);

/* linux/mm/vmscan.c */
extern unsigned long zone_reclaimable_pages(struct zone *zone);
extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
                                        gfp_t gfp_mask, nodemask_t *mask);

#define MEMCG_RECLAIM_MAY_SWAP (1 << 1)
#define MEMCG_RECLAIM_PROACTIVE (1 << 2)
#define MIN_SWAPPINESS 0
#define MAX_SWAPPINESS 200

/* Just reclaim from anon folios in proactive memory reclaim */
#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1)

extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
                                                  unsigned long nr_pages,
                                                  gfp_t gfp_mask,
                                                  unsigned int reclaim_options,
                                                  int *swappiness);
extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem,
                                                gfp_t gfp_mask, bool noswap,
                                                pg_data_t *pgdat,
                                                unsigned long *nr_scanned);
extern unsigned long shrink_all_memory(unsigned long nr_pages);
extern int vm_swappiness;
long remove_mapping(struct address_space *mapping, struct folio *folio);

#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
extern int reclaim_register_node(struct node *node);
extern void reclaim_unregister_node(struct node *node);

#else

static inline int reclaim_register_node(struct node *node)
{
        return 0;
}

static inline void reclaim_unregister_node(struct node *node)
{
}
#endif /* CONFIG_SYSFS && CONFIG_NUMA */

#ifdef CONFIG_NUMA
extern int sysctl_min_unmapped_ratio;
extern int sysctl_min_slab_ratio;
#endif

void check_move_unevictable_folios(struct folio_batch *fbatch);

extern void __meminit kswapd_run(int nid);
extern void __meminit kswapd_stop(int nid);

#ifdef CONFIG_SWAP

int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
                unsigned long nr_pages, sector_t start_block);
int generic_swapfile_activate(struct swap_info_struct *, struct file *,
                sector_t *);

static inline unsigned long total_swapcache_pages(void)
{
        return global_node_page_state(NR_SWAPCACHE);
}

void free_swap_cache(struct folio *folio);
void free_folio_and_swap_cache(struct folio *folio);
void free_pages_and_swap_cache(struct encoded_page **, int);
/* linux/mm/swapfile.c */
extern atomic_long_t nr_swap_pages;
extern long total_swap_pages;
extern atomic_t nr_rotate_swap;

/* Swap 50% full? Release swapcache more aggressively.. */
static inline bool vm_swap_full(void)
{
        return atomic_long_read(&nr_swap_pages) * 2 < total_swap_pages;
}

static inline long get_nr_swap_pages(void)
{
        return atomic_long_read(&nr_swap_pages);
}

extern void si_swapinfo(struct sysinfo *);
int folio_alloc_swap(struct folio *folio, gfp_t gfp_mask);
bool folio_free_swap(struct folio *folio);
void put_swap_folio(struct folio *folio, swp_entry_t entry);
extern swp_entry_t get_swap_page_of_type(int);
extern int add_swap_count_continuation(swp_entry_t, gfp_t);
extern void swap_shmem_alloc(swp_entry_t, int);
extern int swap_duplicate(swp_entry_t);
extern int swapcache_prepare(swp_entry_t entry, int nr);
extern void swap_free_nr(swp_entry_t entry, int nr_pages);
extern void free_swap_and_cache_nr(swp_entry_t entry, int nr);
int swap_type_of(dev_t device, sector_t offset);
int find_first_swap(dev_t *device);
extern unsigned int count_swap_pages(int, int);
extern sector_t swapdev_block(int, pgoff_t);
extern int __swap_count(swp_entry_t entry);
extern bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry);
extern int swp_swapcount(swp_entry_t entry);
struct backing_dev_info;
extern struct swap_info_struct *get_swap_device(swp_entry_t entry);
sector_t swap_folio_sector(struct folio *folio);

static inline void put_swap_device(struct swap_info_struct *si)
{
        percpu_ref_put(&si->users);
}

#else /* CONFIG_SWAP */
static inline struct swap_info_struct *get_swap_device(swp_entry_t entry)
{
        return NULL;
}

static inline void put_swap_device(struct swap_info_struct *si)
{
}

#define get_nr_swap_pages()                        0L
#define total_swap_pages                        0L
#define total_swapcache_pages()                        0UL
#define vm_swap_full()                                0

#define si_swapinfo(val) \
        do { (val)->freeswap = (val)->totalswap = 0; } while (0)
#define free_folio_and_swap_cache(folio) \
        folio_put(folio)
#define free_pages_and_swap_cache(pages, nr) \
        release_pages((pages), (nr));

static inline void free_swap_and_cache_nr(swp_entry_t entry, int nr)
{
}

static inline void free_swap_cache(struct folio *folio)
{
}

static inline int add_swap_count_continuation(swp_entry_t swp, gfp_t gfp_mask)
{
        return 0;
}

static inline void swap_shmem_alloc(swp_entry_t swp, int nr)
{
}

static inline int swap_duplicate(swp_entry_t swp)
{
        return 0;
}

static inline int swapcache_prepare(swp_entry_t swp, int nr)
{
        return 0;
}

static inline void swap_free_nr(swp_entry_t entry, int nr_pages)
{
}

static inline void put_swap_folio(struct folio *folio, swp_entry_t swp)
{
}

static inline int __swap_count(swp_entry_t entry)
{
        return 0;
}

static inline bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry)
{
        return false;
}

static inline int swp_swapcount(swp_entry_t entry)
{
        return 0;
}

static inline int folio_alloc_swap(struct folio *folio, gfp_t gfp_mask)
{
        return -EINVAL;
}

static inline bool folio_free_swap(struct folio *folio)
{
        return false;
}

static inline int add_swap_extent(struct swap_info_struct *sis,
                                  unsigned long start_page,
                                  unsigned long nr_pages, sector_t start_block)
{
        return -EINVAL;
}
#endif /* CONFIG_SWAP */

static inline void free_swap_and_cache(swp_entry_t entry)
{
        free_swap_and_cache_nr(entry, 1);
}

static inline void swap_free(swp_entry_t entry)
{
        swap_free_nr(entry, 1);
}

#ifdef CONFIG_MEMCG
static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
{
        /* Cgroup2 doesn't have per-cgroup swappiness */
        if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
                return READ_ONCE(vm_swappiness);

        /* root ? */
        if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg))
                return READ_ONCE(vm_swappiness);

        return READ_ONCE(memcg->swappiness);
}
#else
static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)
{
        return READ_ONCE(vm_swappiness);
}
#endif

#if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp);
static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
{
        if (mem_cgroup_disabled())
                return;
        __folio_throttle_swaprate(folio, gfp);
}
#else
static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
{
}
#endif

#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP)
int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry);
static inline int mem_cgroup_try_charge_swap(struct folio *folio,
                swp_entry_t entry)
{
        if (mem_cgroup_disabled())
                return 0;
        return __mem_cgroup_try_charge_swap(folio, entry);
}

extern void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
{
        if (mem_cgroup_disabled())
                return;
        __mem_cgroup_uncharge_swap(entry, nr_pages);
}

extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
extern bool mem_cgroup_swap_full(struct folio *folio);
#else
static inline int mem_cgroup_try_charge_swap(struct folio *folio,
                                             swp_entry_t entry)
{
        return 0;
}

static inline void mem_cgroup_uncharge_swap(swp_entry_t entry,
                                            unsigned int nr_pages)
{
}

static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
{
        return get_nr_swap_pages();
}

static inline bool mem_cgroup_swap_full(struct folio *folio)
{
        return vm_swap_full();
}
#endif

#endif /* __KERNEL__*/
#endif /* _LINUX_SWAP_H */
















































































    1 
    1 
    1 


    1 







    1 













    1 





    1 









































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * lib/ts_kmp.c                Knuth-Morris-Pratt text search implementation
 *
 * Authors:        Thomas Graf <tgraf@suug.ch>
 *
 * ==========================================================================
 * 
 *   Implements a linear-time string-matching algorithm due to Knuth,
 *   Morris, and Pratt [1]. Their algorithm avoids the explicit
 *   computation of the transition function DELTA altogether. Its
 *   matching time is O(n), for n being length(text), using just an
 *   auxiliary function PI[1..m], for m being length(pattern),
 *   precomputed from the pattern in time O(m). The array PI allows
 *   the transition function DELTA to be computed efficiently
 *   "on the fly" as needed. Roughly speaking, for any state
 *   "q" = 0,1,...,m and any character "a" in SIGMA, the value
 *   PI["q"] contains the information that is independent of "a" and
 *   is needed to compute DELTA("q", "a") [2]. Since the array PI
 *   has only m entries, whereas DELTA has O(m|SIGMA|) entries, we
 *   save a factor of |SIGMA| in the preprocessing time by computing
 *   PI rather than DELTA.
 *
 *   [1] Cormen, Leiserson, Rivest, Stein
 *       Introdcution to Algorithms, 2nd Edition, MIT Press
 *   [2] See finite automaton theory
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/ctype.h>
#include <linux/textsearch.h>

struct ts_kmp
{
        u8 *                pattern;
        unsigned int        pattern_len;
        unsigned int        prefix_tbl[];
};

static unsigned int kmp_find(struct ts_config *conf, struct ts_state *state)
{
        struct ts_kmp *kmp = ts_config_priv(conf);
        unsigned int i, q = 0, text_len, consumed = state->offset;
        const u8 *text;
        const int icase = conf->flags & TS_IGNORECASE;

        for (;;) {
                text_len = conf->get_next_block(consumed, &text, conf, state);

                if (unlikely(text_len == 0))
                        break;

                for (i = 0; i < text_len; i++) {
                        while (q > 0 && kmp->pattern[q]
                            != (icase ? toupper(text[i]) : text[i]))
                                q = kmp->prefix_tbl[q - 1];
                        if (kmp->pattern[q]
                            == (icase ? toupper(text[i]) : text[i]))
                                q++;
                        if (unlikely(q == kmp->pattern_len)) {
                                state->offset = consumed + i + 1;
                                return state->offset - kmp->pattern_len;
                        }
                }

                consumed += text_len;
        }

        return UINT_MAX;
}

static inline void compute_prefix_tbl(const u8 *pattern, unsigned int len,
                                      unsigned int *prefix_tbl, int flags)
{
        unsigned int k, q;
        const u8 icase = flags & TS_IGNORECASE;

        for (k = 0, q = 1; q < len; q++) {
                while (k > 0 && (icase ? toupper(pattern[k]) : pattern[k])
                    != (icase ? toupper(pattern[q]) : pattern[q]))
                        k = prefix_tbl[k-1];
                if ((icase ? toupper(pattern[k]) : pattern[k])
                    == (icase ? toupper(pattern[q]) : pattern[q]))
                        k++;
                prefix_tbl[q] = k;
        }
}

static struct ts_config *kmp_init(const void *pattern, unsigned int len,
                                  gfp_t gfp_mask, int flags)
{
        struct ts_config *conf;
        struct ts_kmp *kmp;
        int i;
        unsigned int prefix_tbl_len = len * sizeof(unsigned int);
        size_t priv_size = sizeof(*kmp) + len + prefix_tbl_len;

        conf = alloc_ts_config(priv_size, gfp_mask);
        if (IS_ERR(conf))
                return conf;

        conf->flags = flags;
        kmp = ts_config_priv(conf);
        kmp->pattern_len = len;
        compute_prefix_tbl(pattern, len, kmp->prefix_tbl, flags);
        kmp->pattern = (u8 *) kmp->prefix_tbl + prefix_tbl_len;
        if (flags & TS_IGNORECASE)
                for (i = 0; i < len; i++)
                        kmp->pattern[i] = toupper(((u8 *)pattern)[i]);
        else
                memcpy(kmp->pattern, pattern, len);

        return conf;
}

static void *kmp_get_pattern(struct ts_config *conf)
{
        struct ts_kmp *kmp = ts_config_priv(conf);
        return kmp->pattern;
}

static unsigned int kmp_get_pattern_len(struct ts_config *conf)
{
        struct ts_kmp *kmp = ts_config_priv(conf);
        return kmp->pattern_len;
}

static struct ts_ops kmp_ops = {
        .name                  = "kmp",
        .find                  = kmp_find,
        .init                  = kmp_init,
        .get_pattern          = kmp_get_pattern,
        .get_pattern_len  = kmp_get_pattern_len,
        .owner                  = THIS_MODULE,
        .list                  = LIST_HEAD_INIT(kmp_ops.list)
};

static int __init init_kmp(void)
{
        return textsearch_register(&kmp_ops);
}

static void __exit exit_kmp(void)
{
        textsearch_unregister(&kmp_ops);
}

MODULE_DESCRIPTION("Knuth-Morris-Pratt text search implementation");
MODULE_LICENSE("GPL");

module_init(init_kmp);
module_exit(exit_kmp);


















































































































































































































































































































































































































































































































































































































































































































































































    1 

    1 









    3 








    3 



    1 






    1 





    1 


    1 



























    1 





    1 





    1 





















    1 
















































    1 

































    3 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
// SPDX-License-Identifier: GPL-2.0+
/******************************************************************************
 *  speedtch.c  -  Alcatel SpeedTouch USB xDSL modem driver
 *
 *  Copyright (C) 2001, Alcatel
 *  Copyright (C) 2003, Duncan Sands
 *  Copyright (C) 2004, David Woodhouse
 *
 *  Based on "modem_run.c", copyright (C) 2001, Benoit Papillault
 ******************************************************************************/

#include <asm/page.h>
#include <linux/device.h>
#include <linux/errno.h>
#include <linux/firmware.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/slab.h>
#include <linux/stat.h>
#include <linux/timer.h>
#include <linux/types.h>
#include <linux/usb/ch9.h>
#include <linux/workqueue.h>

#include "usbatm.h"

#define DRIVER_AUTHOR        "Johan Verrept, Duncan Sands <duncan.sands@free.fr>"
#define DRIVER_DESC        "Alcatel SpeedTouch USB driver"

static const char speedtch_driver_name[] = "speedtch";

#define CTRL_TIMEOUT 2000        /* milliseconds */
#define DATA_TIMEOUT 2000        /* milliseconds */

#define OFFSET_7        0                /* size 1 */
#define OFFSET_b        1                /* size 8 */
#define OFFSET_d        9                /* size 4 */
#define OFFSET_e        13                /* size 1 */
#define OFFSET_f        14                /* size 1 */

#define SIZE_7                1
#define SIZE_b                8
#define SIZE_d                4
#define SIZE_e                1
#define SIZE_f                1

#define MIN_POLL_DELAY                5000        /* milliseconds */
#define MAX_POLL_DELAY                60000        /* milliseconds */

#define RESUBMIT_DELAY                1000        /* milliseconds */

#define DEFAULT_BULK_ALTSETTING        1
#define DEFAULT_ISOC_ALTSETTING        3
#define DEFAULT_DL_512_FIRST        0
#define DEFAULT_ENABLE_ISOC        0
#define DEFAULT_SW_BUFFERING        0

static unsigned int altsetting = 0; /* zero means: use the default */
static bool dl_512_first = DEFAULT_DL_512_FIRST;
static bool enable_isoc = DEFAULT_ENABLE_ISOC;
static bool sw_buffering = DEFAULT_SW_BUFFERING;

#define DEFAULT_B_MAX_DSL        8128
#define DEFAULT_MODEM_MODE        11
#define MODEM_OPTION_LENGTH        16
static const unsigned char DEFAULT_MODEM_OPTION[MODEM_OPTION_LENGTH] = {
        0x10, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};

static unsigned int BMaxDSL = DEFAULT_B_MAX_DSL;
static unsigned char ModemMode = DEFAULT_MODEM_MODE;
static unsigned char ModemOption[MODEM_OPTION_LENGTH];
static unsigned int num_ModemOption;

module_param(altsetting, uint, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(altsetting,
                "Alternative setting for data interface (bulk_default: "
                __MODULE_STRING(DEFAULT_BULK_ALTSETTING) "; isoc_default: "
                __MODULE_STRING(DEFAULT_ISOC_ALTSETTING) ")");

module_param(dl_512_first, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dl_512_first,
                 "Read 512 bytes before sending firmware (default: "
                 __MODULE_STRING(DEFAULT_DL_512_FIRST) ")");

module_param(enable_isoc, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(enable_isoc,
                "Use isochronous transfers if available (default: "
                __MODULE_STRING(DEFAULT_ENABLE_ISOC) ")");

module_param(sw_buffering, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(sw_buffering,
                 "Enable software buffering (default: "
                 __MODULE_STRING(DEFAULT_SW_BUFFERING) ")");

module_param(BMaxDSL, uint, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(BMaxDSL,
                "default: " __MODULE_STRING(DEFAULT_B_MAX_DSL));

module_param(ModemMode, byte, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(ModemMode,
                "default: " __MODULE_STRING(DEFAULT_MODEM_MODE));

module_param_array(ModemOption, byte, &num_ModemOption, S_IRUGO);
MODULE_PARM_DESC(ModemOption, "default: 0x10,0x00,0x00,0x00,0x20");

#define INTERFACE_DATA                1
#define ENDPOINT_INT                0x81
#define ENDPOINT_BULK_DATA        0x07
#define ENDPOINT_ISOC_DATA        0x07
#define ENDPOINT_FIRMWARE        0x05

struct speedtch_params {
        unsigned int altsetting;
        unsigned int BMaxDSL;
        unsigned char ModemMode;
        unsigned char ModemOption[MODEM_OPTION_LENGTH];
};

struct speedtch_instance_data {
        struct usbatm_data *usbatm;

        struct speedtch_params params; /* set in probe, constant afterwards */

        struct timer_list status_check_timer;
        struct work_struct status_check_work;

        unsigned char last_status;

        int poll_delay; /* milliseconds */

        struct timer_list resubmit_timer;
        struct urb *int_urb;
        unsigned char int_data[16];

        unsigned char scratch_buffer[16];
};

/***************
**  firmware  **
***************/

static void speedtch_set_swbuff(struct speedtch_instance_data *instance, int state)
{
        struct usbatm_data *usbatm = instance->usbatm;
        struct usb_device *usb_dev = usbatm->usb_dev;
        int ret;

        ret = usb_control_msg(usb_dev, usb_sndctrlpipe(usb_dev, 0),
                              0x32, 0x40, state ? 0x01 : 0x00, 0x00, NULL, 0, CTRL_TIMEOUT);
        if (ret < 0)
                usb_warn(usbatm,
                         "%sabling SW buffering: usb_control_msg returned %d\n",
                         state ? "En" : "Dis", ret);
        else
                usb_dbg(usbatm, "speedtch_set_swbuff: %sbled SW buffering\n", state ? "En" : "Dis");
}

static void speedtch_test_sequence(struct speedtch_instance_data *instance)
{
        struct usbatm_data *usbatm = instance->usbatm;
        struct usb_device *usb_dev = usbatm->usb_dev;
        unsigned char *buf = instance->scratch_buffer;
        int ret;

        /* URB 147 */
        buf[0] = 0x1c;
        buf[1] = 0x50;
        ret = usb_control_msg(usb_dev, usb_sndctrlpipe(usb_dev, 0),
                              0x01, 0x40, 0x0b, 0x00, buf, 2, CTRL_TIMEOUT);
        if (ret < 0)
                usb_warn(usbatm, "%s failed on URB147: %d\n", __func__, ret);

        /* URB 148 */
        buf[0] = 0x32;
        buf[1] = 0x00;
        ret = usb_control_msg(usb_dev, usb_sndctrlpipe(usb_dev, 0),
                              0x01, 0x40, 0x02, 0x00, buf, 2, CTRL_TIMEOUT);
        if (ret < 0)
                usb_warn(usbatm, "%s failed on URB148: %d\n", __func__, ret);

        /* URB 149 */
        buf[0] = 0x01;
        buf[1] = 0x00;
        buf[2] = 0x01;
        ret = usb_control_msg(usb_dev, usb_sndctrlpipe(usb_dev, 0),
                              0x01, 0x40, 0x03, 0x00, buf, 3, CTRL_TIMEOUT);
        if (ret < 0)
                usb_warn(usbatm, "%s failed on URB149: %d\n", __func__, ret);

        /* URB 150 */
        buf[0] = 0x01;
        buf[1] = 0x00;
        buf[2] = 0x01;
        ret = usb_control_msg(usb_dev, usb_sndctrlpipe(usb_dev, 0),
                              0x01, 0x40, 0x04, 0x00, buf, 3, CTRL_TIMEOUT);
        if (ret < 0)
                usb_warn(usbatm, "%s failed on URB150: %d\n", __func__, ret);

        /* Extra initialisation in recent drivers - gives higher speeds */

        /* URBext1 */
        buf[0] = instance->params.ModemMode;
        ret = usb_control_msg(usb_dev, usb_sndctrlpipe(usb_dev, 0),
                              0x01, 0x40, 0x11, 0x00, buf, 1, CTRL_TIMEOUT);
        if (ret < 0)
                usb_warn(usbatm, "%s failed on URBext1: %d\n", __func__, ret);

        /* URBext2 */
        /* This seems to be the one which actually triggers the higher sync
           rate -- it does require the new firmware too, although it works OK
           with older firmware */
        ret = usb_control_msg(usb_dev, usb_sndctrlpipe(usb_dev, 0),
                              0x01, 0x40, 0x14, 0x00,
                              instance->params.ModemOption,
                              MODEM_OPTION_LENGTH, CTRL_TIMEOUT);
        if (ret < 0)
                usb_warn(usbatm, "%s failed on URBext2: %d\n", __func__, ret);

        /* URBext3 */
        buf[0] = instance->params.BMaxDSL & 0xff;
        buf[1] = instance->params.BMaxDSL >> 8;
        ret = usb_control_msg(usb_dev, usb_sndctrlpipe(usb_dev, 0),
                              0x01, 0x40, 0x12, 0x00, buf, 2, CTRL_TIMEOUT);
        if (ret < 0)
                usb_warn(usbatm, "%s failed on URBext3: %d\n", __func__, ret);
}

static int speedtch_upload_firmware(struct speedtch_instance_data *instance,
                                     const struct firmware *fw1,
                                     const struct firmware *fw2)
{
        unsigned char *buffer;
        struct usbatm_data *usbatm = instance->usbatm;
        struct usb_device *usb_dev = usbatm->usb_dev;
        int actual_length;
        int ret = 0;
        int offset;

        usb_dbg(usbatm, "%s entered\n", __func__);

        buffer = (unsigned char *)__get_free_page(GFP_KERNEL);
        if (!buffer) {
                ret = -ENOMEM;
                usb_dbg(usbatm, "%s: no memory for buffer!\n", __func__);
                goto out;
        }

        if (!usb_ifnum_to_if(usb_dev, 2)) {
                ret = -ENODEV;
                usb_dbg(usbatm, "%s: interface not found!\n", __func__);
                goto out_free;
        }

        /* URB 7 */
        if (dl_512_first) {        /* some modems need a read before writing the firmware */
                ret = usb_bulk_msg(usb_dev, usb_rcvbulkpipe(usb_dev, ENDPOINT_FIRMWARE),
                                   buffer, 0x200, &actual_length, 2000);

                if (ret < 0 && ret != -ETIMEDOUT)
                        usb_warn(usbatm, "%s: read BLOCK0 from modem failed (%d)!\n", __func__, ret);
                else
                        usb_dbg(usbatm, "%s: BLOCK0 downloaded (%d bytes)\n", __func__, ret);
        }

        /* URB 8 : both leds are static green */
        for (offset = 0; offset < fw1->size; offset += PAGE_SIZE) {
                int thislen = min_t(int, PAGE_SIZE, fw1->size - offset);
                memcpy(buffer, fw1->data + offset, thislen);

                ret = usb_bulk_msg(usb_dev, usb_sndbulkpipe(usb_dev, ENDPOINT_FIRMWARE),
                                   buffer, thislen, &actual_length, DATA_TIMEOUT);

                if (ret < 0) {
                        usb_err(usbatm, "%s: write BLOCK1 to modem failed (%d)!\n", __func__, ret);
                        goto out_free;
                }
                usb_dbg(usbatm, "%s: BLOCK1 uploaded (%zu bytes)\n", __func__, fw1->size);
        }

        /* USB led blinking green, ADSL led off */

        /* URB 11 */
        ret = usb_bulk_msg(usb_dev, usb_rcvbulkpipe(usb_dev, ENDPOINT_FIRMWARE),
                           buffer, 0x200, &actual_length, DATA_TIMEOUT);

        if (ret < 0) {
                usb_err(usbatm, "%s: read BLOCK2 from modem failed (%d)!\n", __func__, ret);
                goto out_free;
        }
        usb_dbg(usbatm, "%s: BLOCK2 downloaded (%d bytes)\n", __func__, actual_length);

        /* URBs 12 to 139 - USB led blinking green, ADSL led off */
        for (offset = 0; offset < fw2->size; offset += PAGE_SIZE) {
                int thislen = min_t(int, PAGE_SIZE, fw2->size - offset);
                memcpy(buffer, fw2->data + offset, thislen);

                ret = usb_bulk_msg(usb_dev, usb_sndbulkpipe(usb_dev, ENDPOINT_FIRMWARE),
                                   buffer, thislen, &actual_length, DATA_TIMEOUT);

                if (ret < 0) {
                        usb_err(usbatm, "%s: write BLOCK3 to modem failed (%d)!\n", __func__, ret);
                        goto out_free;
                }
        }
        usb_dbg(usbatm, "%s: BLOCK3 uploaded (%zu bytes)\n", __func__, fw2->size);

        /* USB led static green, ADSL led static red */

        /* URB 142 */
        ret = usb_bulk_msg(usb_dev, usb_rcvbulkpipe(usb_dev, ENDPOINT_FIRMWARE),
                           buffer, 0x200, &actual_length, DATA_TIMEOUT);

        if (ret < 0) {
                usb_err(usbatm, "%s: read BLOCK4 from modem failed (%d)!\n", __func__, ret);
                goto out_free;
        }

        /* success */
        usb_dbg(usbatm, "%s: BLOCK4 downloaded (%d bytes)\n", __func__, actual_length);

        /* Delay to allow firmware to start up. We can do this here
           because we're in our own kernel thread anyway. */
        msleep_interruptible(1000);

        if ((ret = usb_set_interface(usb_dev, INTERFACE_DATA, instance->params.altsetting)) < 0) {
                usb_err(usbatm, "%s: setting interface to %d failed (%d)!\n", __func__, instance->params.altsetting, ret);
                goto out_free;
        }

        /* Enable software buffering, if requested */
        if (sw_buffering)
                speedtch_set_swbuff(instance, 1);

        /* Magic spell; don't ask us what this does */
        speedtch_test_sequence(instance);

        ret = 0;

out_free:
        free_page((unsigned long)buffer);
out:
        return ret;
}

static int speedtch_find_firmware(struct usbatm_data *usbatm, struct usb_interface *intf,
                                  int phase, const struct firmware **fw_p)
{
        struct device *dev = &intf->dev;
        const u16 bcdDevice = le16_to_cpu(interface_to_usbdev(intf)->descriptor.bcdDevice);
        const u8 major_revision = bcdDevice >> 8;
        const u8 minor_revision = bcdDevice & 0xff;
        char buf[24];

        sprintf(buf, "speedtch-%d.bin.%x.%02x", phase, major_revision, minor_revision);
        usb_dbg(usbatm, "%s: looking for %s\n", __func__, buf);

        if (request_firmware(fw_p, buf, dev)) {
                sprintf(buf, "speedtch-%d.bin.%x", phase, major_revision);
                usb_dbg(usbatm, "%s: looking for %s\n", __func__, buf);

                if (request_firmware(fw_p, buf, dev)) {
                        sprintf(buf, "speedtch-%d.bin", phase);
                        usb_dbg(usbatm, "%s: looking for %s\n", __func__, buf);

                        if (request_firmware(fw_p, buf, dev)) {
                                usb_err(usbatm, "%s: no stage %d firmware found!\n", __func__, phase);
                                return -ENOENT;
                        }
                }
        }

        usb_info(usbatm, "found stage %d firmware %s\n", phase, buf);

        return 0;
}

static int speedtch_heavy_init(struct usbatm_data *usbatm, struct usb_interface *intf)
{
        const struct firmware *fw1, *fw2;
        struct speedtch_instance_data *instance = usbatm->driver_data;
        int ret;

        if ((ret = speedtch_find_firmware(usbatm, intf, 1, &fw1)) < 0)
                return ret;

        if ((ret = speedtch_find_firmware(usbatm, intf, 2, &fw2)) < 0) {
                release_firmware(fw1);
                return ret;
        }

        if ((ret = speedtch_upload_firmware(instance, fw1, fw2)) < 0)
                usb_err(usbatm, "%s: firmware upload failed (%d)!\n", __func__, ret);

        release_firmware(fw2);
        release_firmware(fw1);

        return ret;
}


/**********
**  ATM  **
**********/

static int speedtch_read_status(struct speedtch_instance_data *instance)
{
        struct usbatm_data *usbatm = instance->usbatm;
        struct usb_device *usb_dev = usbatm->usb_dev;
        unsigned char *buf = instance->scratch_buffer;
        int ret;

        memset(buf, 0, 16);

        ret = usb_control_msg(usb_dev, usb_rcvctrlpipe(usb_dev, 0),
                              0x12, 0xc0, 0x07, 0x00, buf + OFFSET_7, SIZE_7,
                              CTRL_TIMEOUT);
        if (ret < 0) {
                atm_dbg(usbatm, "%s: MSG 7 failed\n", __func__);
                return ret;
        }

        ret = usb_control_msg(usb_dev, usb_rcvctrlpipe(usb_dev, 0),
                              0x12, 0xc0, 0x0b, 0x00, buf + OFFSET_b, SIZE_b,
                              CTRL_TIMEOUT);
        if (ret < 0) {
                atm_dbg(usbatm, "%s: MSG B failed\n", __func__);
                return ret;
        }

        ret = usb_control_msg(usb_dev, usb_rcvctrlpipe(usb_dev, 0),
                              0x12, 0xc0, 0x0d, 0x00, buf + OFFSET_d, SIZE_d,
                              CTRL_TIMEOUT);
        if (ret < 0) {
                atm_dbg(usbatm, "%s: MSG D failed\n", __func__);
                return ret;
        }

        ret = usb_control_msg(usb_dev, usb_rcvctrlpipe(usb_dev, 0),
                              0x01, 0xc0, 0x0e, 0x00, buf + OFFSET_e, SIZE_e,
                              CTRL_TIMEOUT);
        if (ret < 0) {
                atm_dbg(usbatm, "%s: MSG E failed\n", __func__);
                return ret;
        }

        ret = usb_control_msg(usb_dev, usb_rcvctrlpipe(usb_dev, 0),
                              0x01, 0xc0, 0x0f, 0x00, buf + OFFSET_f, SIZE_f,
                              CTRL_TIMEOUT);
        if (ret < 0) {
                atm_dbg(usbatm, "%s: MSG F failed\n", __func__);
                return ret;
        }

        return 0;
}

static int speedtch_start_synchro(struct speedtch_instance_data *instance)
{
        struct usbatm_data *usbatm = instance->usbatm;
        struct usb_device *usb_dev = usbatm->usb_dev;
        unsigned char *buf = instance->scratch_buffer;
        int ret;

        atm_dbg(usbatm, "%s entered\n", __func__);

        memset(buf, 0, 2);

        ret = usb_control_msg(usb_dev, usb_rcvctrlpipe(usb_dev, 0),
                              0x12, 0xc0, 0x04, 0x00,
                              buf, 2, CTRL_TIMEOUT);

        if (ret < 0)
                atm_warn(usbatm, "failed to start ADSL synchronisation: %d\n", ret);
        else
                atm_dbg(usbatm, "%s: modem prodded. %d bytes returned: %02x %02x\n",
                        __func__, ret, buf[0], buf[1]);

        return ret;
}

static void speedtch_check_status(struct work_struct *work)
{
        struct speedtch_instance_data *instance =
                container_of(work, struct speedtch_instance_data,
                             status_check_work);
        struct usbatm_data *usbatm = instance->usbatm;
        struct atm_dev *atm_dev = usbatm->atm_dev;
        unsigned char *buf = instance->scratch_buffer;
        int down_speed, up_speed, ret;
        unsigned char status;

#ifdef VERBOSE_DEBUG
        atm_dbg(usbatm, "%s entered\n", __func__);
#endif

        ret = speedtch_read_status(instance);
        if (ret < 0) {
                atm_warn(usbatm, "error %d fetching device status\n", ret);
                instance->poll_delay = min(2 * instance->poll_delay, MAX_POLL_DELAY);
                return;
        }

        instance->poll_delay = max(instance->poll_delay / 2, MIN_POLL_DELAY);

        status = buf[OFFSET_7];

        if ((status != instance->last_status) || !status) {
                atm_dbg(usbatm, "%s: line state 0x%02x\n", __func__, status);

                switch (status) {
                case 0:
                        atm_dev_signal_change(atm_dev, ATM_PHY_SIG_LOST);
                        if (instance->last_status)
                                atm_info(usbatm, "ADSL line is down\n");
                        /* It may never resync again unless we ask it to... */
                        ret = speedtch_start_synchro(instance);
                        break;

                case 0x08:
                        atm_dev_signal_change(atm_dev, ATM_PHY_SIG_UNKNOWN);
                        atm_info(usbatm, "ADSL line is blocked?\n");
                        break;

                case 0x10:
                        atm_dev_signal_change(atm_dev, ATM_PHY_SIG_LOST);
                        atm_info(usbatm, "ADSL line is synchronising\n");
                        break;

                case 0x20:
                        down_speed = buf[OFFSET_b] | (buf[OFFSET_b + 1] << 8)
                                | (buf[OFFSET_b + 2] << 16) | (buf[OFFSET_b + 3] << 24);
                        up_speed = buf[OFFSET_b + 4] | (buf[OFFSET_b + 5] << 8)
                                | (buf[OFFSET_b + 6] << 16) | (buf[OFFSET_b + 7] << 24);

                        if (!(down_speed & 0x0000ffff) && !(up_speed & 0x0000ffff)) {
                                down_speed >>= 16;
                                up_speed >>= 16;
                        }

                        atm_dev->link_rate = down_speed * 1000 / 424;
                        atm_dev_signal_change(atm_dev, ATM_PHY_SIG_FOUND);

                        atm_info(usbatm,
                                 "ADSL line is up (%d kb/s down | %d kb/s up)\n",
                                 down_speed, up_speed);
                        break;

                default:
                        atm_dev_signal_change(atm_dev, ATM_PHY_SIG_UNKNOWN);
                        atm_info(usbatm, "unknown line state %02x\n", status);
                        break;
                }

                instance->last_status = status;
        }
}

static void speedtch_status_poll(struct timer_list *t)
{
        struct speedtch_instance_data *instance = timer_container_of(instance,
                                                                     t,
                                                                     status_check_timer);

        schedule_work(&instance->status_check_work);

        /* The following check is racy, but the race is harmless */
        if (instance->poll_delay < MAX_POLL_DELAY)
                mod_timer(&instance->status_check_timer, jiffies + msecs_to_jiffies(instance->poll_delay));
        else
                atm_warn(instance->usbatm, "Too many failures - disabling line status polling\n");
}

static void speedtch_resubmit_int(struct timer_list *t)
{
        struct speedtch_instance_data *instance = timer_container_of(instance,
                                                                     t,
                                                                     resubmit_timer);
        struct urb *int_urb = instance->int_urb;
        int ret;

        atm_dbg(instance->usbatm, "%s entered\n", __func__);

        if (int_urb) {
                ret = usb_submit_urb(int_urb, GFP_ATOMIC);
                if (!ret)
                        schedule_work(&instance->status_check_work);
                else {
                        atm_dbg(instance->usbatm, "%s: usb_submit_urb failed with result %d\n", __func__, ret);
                        mod_timer(&instance->resubmit_timer, jiffies + msecs_to_jiffies(RESUBMIT_DELAY));
                }
        }
}

static void speedtch_handle_int(struct urb *int_urb)
{
        struct speedtch_instance_data *instance = int_urb->context;
        struct usbatm_data *usbatm = instance->usbatm;
        unsigned int count = int_urb->actual_length;
        int status = int_urb->status;
        int ret;

        /* The magic interrupt for "up state" */
        static const unsigned char up_int[6]   = { 0xa1, 0x00, 0x01, 0x00, 0x00, 0x00 };
        /* The magic interrupt for "down state" */
        static const unsigned char down_int[6] = { 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00 };

        atm_dbg(usbatm, "%s entered\n", __func__);

        if (status < 0) {
                atm_dbg(usbatm, "%s: nonzero urb status %d!\n", __func__, status);
                goto fail;
        }

        if ((count == 6) && !memcmp(up_int, instance->int_data, 6)) {
                timer_delete(&instance->status_check_timer);
                atm_info(usbatm, "DSL line goes up\n");
        } else if ((count == 6) && !memcmp(down_int, instance->int_data, 6)) {
                atm_info(usbatm, "DSL line goes down\n");
        } else {
                int i;

                atm_dbg(usbatm, "%s: unknown interrupt packet of length %d:", __func__, count);
                for (i = 0; i < count; i++)
                        printk(" %02x", instance->int_data[i]);
                printk("\n");
                goto fail;
        }

        int_urb = instance->int_urb;
        if (int_urb) {
                ret = usb_submit_urb(int_urb, GFP_ATOMIC);
                schedule_work(&instance->status_check_work);
                if (ret < 0) {
                        atm_dbg(usbatm, "%s: usb_submit_urb failed with result %d\n", __func__, ret);
                        goto fail;
                }
        }

        return;

fail:
        int_urb = instance->int_urb;
        if (int_urb)
                mod_timer(&instance->resubmit_timer, jiffies + msecs_to_jiffies(RESUBMIT_DELAY));
}

static int speedtch_atm_start(struct usbatm_data *usbatm, struct atm_dev *atm_dev)
{
        struct usb_device *usb_dev = usbatm->usb_dev;
        struct speedtch_instance_data *instance = usbatm->driver_data;
        int i, ret;
        unsigned char mac_str[13];

        atm_dbg(usbatm, "%s entered\n", __func__);

        /* Set MAC address, it is stored in the serial number */
        memset(atm_dev->esi, 0, sizeof(atm_dev->esi));
        if (usb_string(usb_dev, usb_dev->descriptor.iSerialNumber, mac_str, sizeof(mac_str)) == 12) {
                for (i = 0; i < 6; i++)
                        atm_dev->esi[i] = (hex_to_bin(mac_str[i * 2]) << 4) +
                                hex_to_bin(mac_str[i * 2 + 1]);
        }

        /* Start modem synchronisation */
        ret = speedtch_start_synchro(instance);

        /* Set up interrupt endpoint */
        if (instance->int_urb) {
                ret = usb_submit_urb(instance->int_urb, GFP_KERNEL);
                if (ret < 0) {
                        /* Doesn't matter; we'll poll anyway */
                        atm_dbg(usbatm, "%s: submission of interrupt URB failed (%d)!\n", __func__, ret);
                        usb_free_urb(instance->int_urb);
                        instance->int_urb = NULL;
                }
        }

        /* Start status polling */
        mod_timer(&instance->status_check_timer, jiffies + msecs_to_jiffies(1000));

        return 0;
}

static void speedtch_atm_stop(struct usbatm_data *usbatm, struct atm_dev *atm_dev)
{
        struct speedtch_instance_data *instance = usbatm->driver_data;
        struct urb *int_urb = instance->int_urb;

        atm_dbg(usbatm, "%s entered\n", __func__);

        timer_delete_sync(&instance->status_check_timer);

        /*
         * Since resubmit_timer and int_urb can schedule themselves and
         * each other, shutting them down correctly takes some care
         */
        instance->int_urb = NULL; /* signal shutdown */
        mb();
        usb_kill_urb(int_urb);
        timer_delete_sync(&instance->resubmit_timer);
        /*
         * At this point, speedtch_handle_int and speedtch_resubmit_int
         * can run or be running, but instance->int_urb == NULL means that
         * they will not reschedule
         */
        usb_kill_urb(int_urb);
        timer_delete_sync(&instance->resubmit_timer);
        usb_free_urb(int_urb);

        flush_work(&instance->status_check_work);
}

static int speedtch_pre_reset(struct usb_interface *intf)
{
        return 0;
}

static int speedtch_post_reset(struct usb_interface *intf)
{
        return 0;
}


/**********
**  USB  **
**********/

static const struct usb_device_id speedtch_usb_ids[] = {
        {USB_DEVICE(0x06b9, 0x4061)},
        {}
};

MODULE_DEVICE_TABLE(usb, speedtch_usb_ids);

static int speedtch_usb_probe(struct usb_interface *, const struct usb_device_id *);

static struct usb_driver speedtch_usb_driver = {
        .name                = speedtch_driver_name,
        .probe                = speedtch_usb_probe,
        .disconnect        = usbatm_usb_disconnect,
        .pre_reset        = speedtch_pre_reset,
        .post_reset        = speedtch_post_reset,
        .id_table        = speedtch_usb_ids
};

static void speedtch_release_interfaces(struct usb_device *usb_dev,
                                        int num_interfaces)
{
        struct usb_interface *cur_intf;
        int i;

        for (i = 0; i < num_interfaces; i++) {
                cur_intf = usb_ifnum_to_if(usb_dev, i);
                if (cur_intf) {
                        usb_set_intfdata(cur_intf, NULL);
                        usb_driver_release_interface(&speedtch_usb_driver, cur_intf);
                }
        }
}

static int speedtch_bind(struct usbatm_data *usbatm,
                         struct usb_interface *intf,
                         const struct usb_device_id *id)
{
        struct usb_device *usb_dev = interface_to_usbdev(intf);
        struct usb_interface *cur_intf, *data_intf;
        struct speedtch_instance_data *instance;
        int ifnum = intf->altsetting->desc.bInterfaceNumber;
        int num_interfaces = usb_dev->actconfig->desc.bNumInterfaces;
        int i, ret;
        int use_isoc;

        usb_dbg(usbatm, "%s entered\n", __func__);

        /* sanity checks */

        if (usb_dev->descriptor.bDeviceClass != USB_CLASS_VENDOR_SPEC) {
                usb_err(usbatm, "%s: wrong device class %d\n", __func__, usb_dev->descriptor.bDeviceClass);
                return -ENODEV;
        }

        data_intf = usb_ifnum_to_if(usb_dev, INTERFACE_DATA);
        if (!data_intf) {
                usb_err(usbatm, "%s: data interface not found!\n", __func__);
                return -ENODEV;
        }

        /* claim all interfaces */

        for (i = 0; i < num_interfaces; i++) {
                cur_intf = usb_ifnum_to_if(usb_dev, i);

                if ((i != ifnum) && cur_intf) {
                        ret = usb_driver_claim_interface(&speedtch_usb_driver, cur_intf, usbatm);

                        if (ret < 0) {
                                usb_err(usbatm, "%s: failed to claim interface %2d (%d)!\n", __func__, i, ret);
                                speedtch_release_interfaces(usb_dev, i);
                                return ret;
                        }
                }
        }

        instance = kzalloc(sizeof(*instance), GFP_KERNEL);

        if (!instance) {
                ret = -ENOMEM;
                goto fail_release;
        }

        instance->usbatm = usbatm;

        /* module parameters may change at any moment, so take a snapshot */
        instance->params.altsetting = altsetting;
        instance->params.BMaxDSL = BMaxDSL;
        instance->params.ModemMode = ModemMode;
        memcpy(instance->params.ModemOption, DEFAULT_MODEM_OPTION, MODEM_OPTION_LENGTH);
        memcpy(instance->params.ModemOption, ModemOption, num_ModemOption);
        use_isoc = enable_isoc;

        if (instance->params.altsetting)
                if ((ret = usb_set_interface(usb_dev, INTERFACE_DATA, instance->params.altsetting)) < 0) {
                        usb_err(usbatm, "%s: setting interface to %2d failed (%d)!\n", __func__, instance->params.altsetting, ret);
                        instance->params.altsetting = 0; /* fall back to default */
                }

        if (!instance->params.altsetting && use_isoc)
                if ((ret = usb_set_interface(usb_dev, INTERFACE_DATA, DEFAULT_ISOC_ALTSETTING)) < 0) {
                        usb_dbg(usbatm, "%s: setting interface to %2d failed (%d)!\n", __func__, DEFAULT_ISOC_ALTSETTING, ret);
                        use_isoc = 0; /* fall back to bulk */
                }

        if (use_isoc) {
                const struct usb_host_interface *desc = data_intf->cur_altsetting;
                const __u8 target_address = USB_DIR_IN | usbatm->driver->isoc_in;

                use_isoc = 0; /* fall back to bulk if endpoint not found */

                for (i = 0; i < desc->desc.bNumEndpoints; i++) {
                        const struct usb_endpoint_descriptor *endpoint_desc = &desc->endpoint[i].desc;

                        if ((endpoint_desc->bEndpointAddress == target_address)) {
                                use_isoc =
                                        usb_endpoint_xfer_isoc(endpoint_desc);
                                break;
                        }
                }

                if (!use_isoc)
                        usb_info(usbatm, "isochronous transfer not supported - using bulk\n");
        }

        if (!use_isoc && !instance->params.altsetting)
                if ((ret = usb_set_interface(usb_dev, INTERFACE_DATA, DEFAULT_BULK_ALTSETTING)) < 0) {
                        usb_err(usbatm, "%s: setting interface to %2d failed (%d)!\n", __func__, DEFAULT_BULK_ALTSETTING, ret);
                        goto fail_free;
                }

        if (!instance->params.altsetting)
                instance->params.altsetting = use_isoc ? DEFAULT_ISOC_ALTSETTING : DEFAULT_BULK_ALTSETTING;

        usbatm->flags |= (use_isoc ? UDSL_USE_ISOC : 0);

        INIT_WORK(&instance->status_check_work, speedtch_check_status);
        timer_setup(&instance->status_check_timer, speedtch_status_poll, 0);
        instance->last_status = 0xff;
        instance->poll_delay = MIN_POLL_DELAY;

        timer_setup(&instance->resubmit_timer, speedtch_resubmit_int, 0);

        instance->int_urb = usb_alloc_urb(0, GFP_KERNEL);

        if (instance->int_urb)
                usb_fill_int_urb(instance->int_urb, usb_dev,
                                 usb_rcvintpipe(usb_dev, ENDPOINT_INT),
                                 instance->int_data, sizeof(instance->int_data),
                                 speedtch_handle_int, instance, 16);
        else
                usb_dbg(usbatm, "%s: no memory for interrupt urb!\n", __func__);

        /* check whether the modem already seems to be alive */
        ret = usb_control_msg(usb_dev, usb_rcvctrlpipe(usb_dev, 0),
                              0x12, 0xc0, 0x07, 0x00,
                              instance->scratch_buffer + OFFSET_7, SIZE_7, 500);

        usbatm->flags |= (ret == SIZE_7 ? UDSL_SKIP_HEAVY_INIT : 0);

        usb_dbg(usbatm, "%s: firmware %s loaded\n", __func__, usbatm->flags & UDSL_SKIP_HEAVY_INIT ? "already" : "not");

        if (!(usbatm->flags & UDSL_SKIP_HEAVY_INIT))
                if ((ret = usb_reset_device(usb_dev)) < 0) {
                        usb_err(usbatm, "%s: device reset failed (%d)!\n", __func__, ret);
                        goto fail_free;
                }

        usbatm->driver_data = instance;

        return 0;

fail_free:
        usb_free_urb(instance->int_urb);
        kfree(instance);
fail_release:
        speedtch_release_interfaces(usb_dev, num_interfaces);
        return ret;
}

static void speedtch_unbind(struct usbatm_data *usbatm, struct usb_interface *intf)
{
        struct usb_device *usb_dev = interface_to_usbdev(intf);
        struct speedtch_instance_data *instance = usbatm->driver_data;

        usb_dbg(usbatm, "%s entered\n", __func__);

        speedtch_release_interfaces(usb_dev, usb_dev->actconfig->desc.bNumInterfaces);
        usb_free_urb(instance->int_urb);
        kfree(instance);
}


/***********
**  init  **
***********/

static struct usbatm_driver speedtch_usbatm_driver = {
        .driver_name        = speedtch_driver_name,
        .bind                = speedtch_bind,
        .heavy_init        = speedtch_heavy_init,
        .unbind                = speedtch_unbind,
        .atm_start        = speedtch_atm_start,
        .atm_stop        = speedtch_atm_stop,
        .bulk_in        = ENDPOINT_BULK_DATA,
        .bulk_out        = ENDPOINT_BULK_DATA,
        .isoc_in        = ENDPOINT_ISOC_DATA
};

static int speedtch_usb_probe(struct usb_interface *intf, const struct usb_device_id *id)
{
        return usbatm_usb_probe(intf, id, &speedtch_usbatm_driver);
}

module_usb_driver(speedtch_usb_driver);

MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);
MODULE_LICENSE("GPL");


































































    4 






    4 
    4 
















































    4 









    2 
    2 



























































































































































































    4 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * STV0680 USB Camera Driver
 *
 * Copyright (C) 2009 Hans de Goede <hdegoede@redhat.com>
 *
 * This module is adapted from the in kernel v4l1 stv680 driver:
 *
 *  STV0680 USB Camera Driver, by Kevin Sisson (kjsisson@bellsouth.net)
 *
 * Thanks to STMicroelectronics for information on the usb commands, and
 * to Steve Miller at STM for his help and encouragement while I was
 * writing this driver.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#define MODULE_NAME "stv0680"

#include "gspca.h"

MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
MODULE_DESCRIPTION("STV0680 USB Camera Driver");
MODULE_LICENSE("GPL");

/* specific webcam descriptor */
struct sd {
        struct gspca_dev gspca_dev;                /* !! must be the first item */
        struct v4l2_pix_format mode;
        u8 orig_mode;
        u8 video_mode;
        u8 current_mode;
};

static int stv_sndctrl(struct gspca_dev *gspca_dev, int set, u8 req, u16 val,
                       int size)
{
        int ret;
        u8 req_type = 0;
        unsigned int pipe = 0;

        switch (set) {
        case 0: /*  0xc1  */
                req_type = USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_ENDPOINT;
                pipe = usb_rcvctrlpipe(gspca_dev->dev, 0);
                break;
        case 1: /*  0x41  */
                req_type = USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_ENDPOINT;
                pipe = usb_sndctrlpipe(gspca_dev->dev, 0);
                break;
        case 2:        /*  0x80  */
                req_type = USB_DIR_IN | USB_RECIP_DEVICE;
                pipe = usb_rcvctrlpipe(gspca_dev->dev, 0);
                break;
        case 3:        /*  0x40  */
                req_type = USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE;
                pipe = usb_sndctrlpipe(gspca_dev->dev, 0);
                break;
        }

        ret = usb_control_msg(gspca_dev->dev, pipe,
                              req, req_type,
                              val, 0, gspca_dev->usb_buf, size, 500);

        if ((ret < 0) && (req != 0x0a))
                pr_err("usb_control_msg error %i, request = 0x%x, error = %i\n",
                       set, req, ret);

        return ret;
}

static int stv0680_handle_error(struct gspca_dev *gspca_dev, int ret)
{
        stv_sndctrl(gspca_dev, 0, 0x80, 0, 0x02); /* Get Last Error */
        gspca_err(gspca_dev, "last error: %i,  command = 0x%x\n",
                  gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
        return ret;
}

static int stv0680_get_video_mode(struct gspca_dev *gspca_dev)
{
        /* Note not sure if this init of usb_buf is really necessary */
        memset(gspca_dev->usb_buf, 0, 8);
        gspca_dev->usb_buf[0] = 0x0f;

        if (stv_sndctrl(gspca_dev, 0, 0x87, 0, 0x08) != 0x08) {
                gspca_err(gspca_dev, "Get_Camera_Mode failed\n");
                return stv0680_handle_error(gspca_dev, -EIO);
        }

        return gspca_dev->usb_buf[0]; /* 01 = VGA, 03 = QVGA, 00 = CIF */
}

static int stv0680_set_video_mode(struct gspca_dev *gspca_dev, u8 mode)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (sd->current_mode == mode)
                return 0;

        memset(gspca_dev->usb_buf, 0, 8);
        gspca_dev->usb_buf[0] = mode;

        if (stv_sndctrl(gspca_dev, 3, 0x07, 0x0100, 0x08) != 0x08) {
                gspca_err(gspca_dev, "Set_Camera_Mode failed\n");
                return stv0680_handle_error(gspca_dev, -EIO);
        }

        /* Verify we got what we've asked for */
        if (stv0680_get_video_mode(gspca_dev) != mode) {
                gspca_err(gspca_dev, "Error setting camera video mode!\n");
                return -EIO;
        }

        sd->current_mode = mode;

        return 0;
}

/* this function is called at probe time */
static int sd_config(struct gspca_dev *gspca_dev,
                        const struct usb_device_id *id)
{
        int ret;
        struct sd *sd = (struct sd *) gspca_dev;
        struct cam *cam = &gspca_dev->cam;

        /* Give the camera some time to settle, otherwise initialization will
           fail on hotplug, and yes it really needs a full second. */
        msleep(1000);

        /* ping camera to be sure STV0680 is present */
        if (stv_sndctrl(gspca_dev, 0, 0x88, 0x5678, 0x02) != 0x02 ||
            gspca_dev->usb_buf[0] != 0x56 || gspca_dev->usb_buf[1] != 0x78) {
                gspca_err(gspca_dev, "STV(e): camera ping failed!!\n");
                return stv0680_handle_error(gspca_dev, -ENODEV);
        }

        /* get camera descriptor */
        if (stv_sndctrl(gspca_dev, 2, 0x06, 0x0200, 0x09) != 0x09)
                return stv0680_handle_error(gspca_dev, -ENODEV);

        if (stv_sndctrl(gspca_dev, 2, 0x06, 0x0200, 0x22) != 0x22 ||
            gspca_dev->usb_buf[7] != 0xa0 || gspca_dev->usb_buf[8] != 0x23) {
                gspca_err(gspca_dev, "Could not get descriptor 0200\n");
                return stv0680_handle_error(gspca_dev, -ENODEV);
        }
        if (stv_sndctrl(gspca_dev, 0, 0x8a, 0, 0x02) != 0x02)
                return stv0680_handle_error(gspca_dev, -ENODEV);
        if (stv_sndctrl(gspca_dev, 0, 0x8b, 0, 0x24) != 0x24)
                return stv0680_handle_error(gspca_dev, -ENODEV);
        if (stv_sndctrl(gspca_dev, 0, 0x85, 0, 0x10) != 0x10)
                return stv0680_handle_error(gspca_dev, -ENODEV);

        if (!(gspca_dev->usb_buf[7] & 0x09)) {
                gspca_err(gspca_dev, "Camera supports neither CIF nor QVGA mode\n");
                return -ENODEV;
        }
        if (gspca_dev->usb_buf[7] & 0x01)
                gspca_dbg(gspca_dev, D_PROBE, "Camera supports CIF mode\n");
        if (gspca_dev->usb_buf[7] & 0x02)
                gspca_dbg(gspca_dev, D_PROBE, "Camera supports VGA mode\n");
        if (gspca_dev->usb_buf[7] & 0x04)
                gspca_dbg(gspca_dev, D_PROBE, "Camera supports QCIF mode\n");
        if (gspca_dev->usb_buf[7] & 0x08)
                gspca_dbg(gspca_dev, D_PROBE, "Camera supports QVGA mode\n");

        if (gspca_dev->usb_buf[7] & 0x01)
                sd->video_mode = 0x00; /* CIF */
        else
                sd->video_mode = 0x03; /* QVGA */

        /* FW rev, ASIC rev, sensor ID  */
        gspca_dbg(gspca_dev, D_PROBE, "Firmware rev is %i.%i\n",
                  gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
        gspca_dbg(gspca_dev, D_PROBE, "ASIC rev is %i.%i",
                  gspca_dev->usb_buf[2], gspca_dev->usb_buf[3]);
        gspca_dbg(gspca_dev, D_PROBE, "Sensor ID is %i",
                  (gspca_dev->usb_buf[4]*16) + (gspca_dev->usb_buf[5]>>4));


        ret = stv0680_get_video_mode(gspca_dev);
        if (ret < 0)
                return ret;
        sd->current_mode = sd->orig_mode = ret;

        ret = stv0680_set_video_mode(gspca_dev, sd->video_mode);
        if (ret < 0)
                return ret;

        /* Get mode details */
        if (stv_sndctrl(gspca_dev, 0, 0x8f, 0, 0x10) != 0x10)
                return stv0680_handle_error(gspca_dev, -EIO);

        cam->bulk = 1;
        cam->bulk_nurbs = 1; /* The cam cannot handle more */
        cam->bulk_size = (gspca_dev->usb_buf[0] << 24) |
                         (gspca_dev->usb_buf[1] << 16) |
                         (gspca_dev->usb_buf[2] << 8) |
                         (gspca_dev->usb_buf[3]);
        sd->mode.width = (gspca_dev->usb_buf[4] << 8) |
                         (gspca_dev->usb_buf[5]);  /* 322, 356, 644 */
        sd->mode.height = (gspca_dev->usb_buf[6] << 8) |
                          (gspca_dev->usb_buf[7]); /* 242, 292, 484 */
        sd->mode.pixelformat = V4L2_PIX_FMT_STV0680;
        sd->mode.field = V4L2_FIELD_NONE;
        sd->mode.bytesperline = sd->mode.width;
        sd->mode.sizeimage = cam->bulk_size;
        sd->mode.colorspace = V4L2_COLORSPACE_SRGB;

        /* origGain = gspca_dev->usb_buf[12]; */

        cam->cam_mode = &sd->mode;
        cam->nmodes = 1;


        ret = stv0680_set_video_mode(gspca_dev, sd->orig_mode);
        if (ret < 0)
                return ret;

        if (stv_sndctrl(gspca_dev, 2, 0x06, 0x0100, 0x12) != 0x12 ||
            gspca_dev->usb_buf[8] != 0x53 || gspca_dev->usb_buf[9] != 0x05) {
                pr_err("Could not get descriptor 0100\n");
                return stv0680_handle_error(gspca_dev, -EIO);
        }

        return 0;
}

/* this function is called at probe and resume time */
static int sd_init(struct gspca_dev *gspca_dev)
{
        return 0;
}

/* -- start the camera -- */
static int sd_start(struct gspca_dev *gspca_dev)
{
        int ret;
        struct sd *sd = (struct sd *) gspca_dev;

        ret = stv0680_set_video_mode(gspca_dev, sd->video_mode);
        if (ret < 0)
                return ret;

        if (stv_sndctrl(gspca_dev, 0, 0x85, 0, 0x10) != 0x10)
                return stv0680_handle_error(gspca_dev, -EIO);

        /* Start stream at:
           0x0000 = CIF (352x288)
           0x0100 = VGA (640x480)
           0x0300 = QVGA (320x240) */
        if (stv_sndctrl(gspca_dev, 1, 0x09, sd->video_mode << 8, 0x0) != 0x0)
                return stv0680_handle_error(gspca_dev, -EIO);

        return 0;
}

static void sd_stopN(struct gspca_dev *gspca_dev)
{
        /* This is a high priority command; it stops all lower order cmds */
        if (stv_sndctrl(gspca_dev, 1, 0x04, 0x0000, 0x0) != 0x0)
                stv0680_handle_error(gspca_dev, -EIO);
}

static void sd_stop0(struct gspca_dev *gspca_dev)
{
        struct sd *sd = (struct sd *) gspca_dev;

        if (!sd->gspca_dev.present)
                return;

        stv0680_set_video_mode(gspca_dev, sd->orig_mode);
}

static void sd_pkt_scan(struct gspca_dev *gspca_dev,
                        u8 *data,
                        int len)
{
        struct sd *sd = (struct sd *) gspca_dev;

        /* Every now and then the camera sends a 16 byte packet, no idea
           what it contains, but it is not image data, when this
           happens the frame received before this packet is corrupt,
           so discard it. */
        if (len != sd->mode.sizeimage) {
                gspca_dev->last_packet_type = DISCARD_PACKET;
                return;
        }

        /* Finish the previous frame, we do this upon reception of the next
           packet, even though it is already complete so that the strange 16
           byte packets send after a corrupt frame can discard it. */
        gspca_frame_add(gspca_dev, LAST_PACKET, NULL, 0);

        /* Store the just received frame */
        gspca_frame_add(gspca_dev, FIRST_PACKET, data, len);
}

/* sub-driver description */
static const struct sd_desc sd_desc = {
        .name = MODULE_NAME,
        .config = sd_config,
        .init = sd_init,
        .start = sd_start,
        .stopN = sd_stopN,
        .stop0 = sd_stop0,
        .pkt_scan = sd_pkt_scan,
};

/* -- module initialisation -- */
static const struct usb_device_id device_table[] = {
        {USB_DEVICE(0x0553, 0x0202)},
        {USB_DEVICE(0x041e, 0x4007)},
        {}
};
MODULE_DEVICE_TABLE(usb, device_table);

/* -- device connect -- */
static int sd_probe(struct usb_interface *intf,
                        const struct usb_device_id *id)
{
        return gspca_dev_probe(intf, id, &sd_desc, sizeof(struct sd),
                                THIS_MODULE);
}

static struct usb_driver sd_driver = {
        .name = MODULE_NAME,
        .id_table = device_table,
        .probe = sd_probe,
        .disconnect = gspca_disconnect,
#ifdef CONFIG_PM
        .suspend = gspca_suspend,
        .resume = gspca_resume,
        .reset_resume = gspca_resume,
#endif
};

module_usb_driver(sd_driver);



























































































































































 5060 
 5073 




































   14 







   14 



















   16 
















   16 


   16 







  199 




















 11484 
























 11349 





 11356 




 11362 

 11351 



 11358 




 11490 



 11484 


 11358 
 11390 
 11355 





 17909 


 17927 

 12975 
 11487 

 17918 





 12882 




   77 
    1 
   68 

   58 



   77 






  649 
  359 
    2 
  667 



 12677 
 12680 







 12686 




 17944 


 17947 



    1 







    1 
    1 
    1 










  765 




  766 




   77 


   76 




   76 






   32 






    9 











    9 
 1002 
 9405 


















 8468 













 12899 



 12941 








 11329 





 12874 



 12874 

    3 
 12874 
 12879 








   91 





   90 



   90 

   91 

   76 








 11296 






    1 





 11288 

 11329 






    4 

    4 




 11290 















  483 

  427 
  427 



  475 
   14 


  194 
  305 











  611 









  610 















  429 









    2 


    1 





  423 
    4 




    4 

  422 





  426 

    4 






  423 





























  482 



    1 


  483 







  191 
  307 




  102 


  406 
   10 
   11 

  194 

  305 






   86 
  427 









  473 
    4 
   16 







  483 

















  483 






 11131 










 11129 

 11123 




 1184 











 11139 


















   90 
 10385 
















 3373 
 10452 













   34 
   33 














 8990 
 8991 




   57 
   67 
   56 








    4 
   23 
   57 




   96 













   16 







   89 











    8 



    6 


    6 
    5 
    1 




   75 








    5 


    2 




    6 
   62 


   60 



   22 
   44 



   57 



   11 



   11 



   52 
   23 


   50 










    8 

   12 
   38 
   46 
    4 




   55 

   51 


   51 









    7 





   17 





    1 


    2 

    2 








    2 


   10 
    2 






    4 


    4 












   24 









    5 





   19 
   19 






  106 











    3 





 12874 

   10 






   14 




    4 



    2 



    2 


    7 

   11 


 12870 





 12620 








  294 

















 9201 







 9197 

 9209 






   18 

   18 




    5 



    5 
    1 

    5 

    5 

    5 

    5 








 1062 


  805 







 9181 


 9184 




    3 

    3 



 18662 


 18676 
 18673 

 18628 


 2738 



  452 



















 12915 

 12882 





 12814 





 12885 






 12895 

 12875 





 12915 




   20 




   20 




 9190 




 8185 





 8190 
 8187 


 8187 

 8186 






  407 












   62 

 5319 





























    3 


 8048 



 5332 
 3043 


    3 


    1 









 5335 
    8 

 5174 

  306 




  306 


 5335 








 5332 






    1 




 5333 
   11 
 5338 










 5327 







 9053 
























 7120 
 8660 



 9052 

 8651 
   48 




 4306 
 5305 
   32 








 6630 











































































   33 




   33 
   33 





   72 





    3 


   69 



   46 







   43 




    5 

   17 

   19 






   20 
   13 





    2 

    4 
    2 
    2 




    3 





    2 


    2 


    6 




   17 





   35 





    2 


    1 

    2 


    5 





    2 


    2 




    8 





    1 












    3 


    2 


    5 


    2 




    8 





    1 
    9 







    4 








    4 



    1 



 12886 










    4 



  178 



 12922 
   10 




    1 

    2 




  386 





 12706 






 11103 



 2271 



    3 












 12886 










 12823 
   57 





 12818 





  670 







    1 





   56 





  659 










    2 











  519 
  293 





  367 

  306 






  659 
    4 
  659 
    4 



  580 
   87 



  395 


  364 
    7 






  669 

















  199 



















  199 










  199 




















  199 




















































































 6216 


















 9694 


 9682 



    3 
   10 




    3 




    3 
    7 

















   11 
    1 
   10 
   11 



    2 


    2 
    7 

    2 
    7 

    8 
    1 

    9 












 1071 








 1071 



 1067 

    4 
    4 









 1669 











 1449 
  369 
    6 













 1496 





  282 



 1502 





    2 













 1665 




 1662 
 1664 

 1675 












 1609 
   69 




 1648 




    1 

  805 











 1070 






    2 

 1068 

  554 
  519 


















 1588 





 1584 










    2 






















  735 



  718 








 1431 










  145 

  147 









 8726 












 1335 
 7905 



















 1337 
 7923 


 7902 



 8721 
    7 


















 11085 




 11005 






  316 



   16 



   24 



 10963 



 2838 
 7802 























 3755 





 3388 
  761 










 3383 


  761 

















   17 





    3 




   18 


















   17 


   17 

   18 






   18 








   18 






   18 








   17 


    3 






   18 










   17 




   17 




























   18 






   18 








   18 



    3 


   17 



   18 




























































  199 




























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * NETLINK      Kernel-user communication protocol.
 *
 *                 Authors:        Alan Cox <alan@lxorguk.ukuu.org.uk>
 *                                 Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
 *                                 Patrick McHardy <kaber@trash.net>
 *
 * Tue Jun 26 14:36:48 MEST 2001 Herbert "herp" Rosmanith
 *                               added netlink_proto_exit
 * Tue Jan 22 18:32:44 BRST 2002 Arnaldo C. de Melo <acme@conectiva.com.br>
 *                                  use nlk_sk, as sk->protinfo is on a diet 8)
 * Fri Jul 22 19:51:12 MEST 2005 Harald Welte <laforge@gnumonks.org>
 *                                  - inc module use count of module that owns
 *                                    the kernel socket in case userspace opens
 *                                    socket of same protocol
 *                                  - remove all module support, since netlink is
 *                                    mandatory if CONFIG_NET=y these days
 */

#include <linux/module.h>

#include <linux/bpf.h>
#include <linux/capability.h>
#include <linux/kernel.h>
#include <linux/filter.h>
#include <linux/init.h>
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/stat.h>
#include <linux/socket.h>
#include <linux/un.h>
#include <linux/fcntl.h>
#include <linux/termios.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/notifier.h>
#include <linux/security.h>
#include <linux/jhash.h>
#include <linux/jiffies.h>
#include <linux/random.h>
#include <linux/bitops.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <linux/audit.h>
#include <linux/mutex.h>
#include <linux/vmalloc.h>
#include <linux/if_arp.h>
#include <linux/rhashtable.h>
#include <asm/cacheflush.h>
#include <linux/hash.h>
#include <linux/net_namespace.h>
#include <linux/nospec.h>
#include <linux/btf_ids.h>

#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/sock.h>
#include <net/scm.h>
#include <net/netlink.h>
#define CREATE_TRACE_POINTS
#include <trace/events/netlink.h>

#include "af_netlink.h"
#include "genetlink.h"

struct listeners {
        struct rcu_head                rcu;
        unsigned long                masks[];
};

/* state bits */
#define NETLINK_S_CONGESTED                0x0

static inline int netlink_is_kernel(struct sock *sk)
{
        return nlk_test_bit(KERNEL_SOCKET, sk);
}

struct netlink_table *nl_table __read_mostly;
EXPORT_SYMBOL_GPL(nl_table);

static DECLARE_WAIT_QUEUE_HEAD(nl_table_wait);

static struct lock_class_key nlk_cb_mutex_keys[MAX_LINKS];

static const char *const nlk_cb_mutex_key_strings[MAX_LINKS + 1] = {
        "nlk_cb_mutex-ROUTE",
        "nlk_cb_mutex-1",
        "nlk_cb_mutex-USERSOCK",
        "nlk_cb_mutex-FIREWALL",
        "nlk_cb_mutex-SOCK_DIAG",
        "nlk_cb_mutex-NFLOG",
        "nlk_cb_mutex-XFRM",
        "nlk_cb_mutex-SELINUX",
        "nlk_cb_mutex-ISCSI",
        "nlk_cb_mutex-AUDIT",
        "nlk_cb_mutex-FIB_LOOKUP",
        "nlk_cb_mutex-CONNECTOR",
        "nlk_cb_mutex-NETFILTER",
        "nlk_cb_mutex-IP6_FW",
        "nlk_cb_mutex-DNRTMSG",
        "nlk_cb_mutex-KOBJECT_UEVENT",
        "nlk_cb_mutex-GENERIC",
        "nlk_cb_mutex-17",
        "nlk_cb_mutex-SCSITRANSPORT",
        "nlk_cb_mutex-ECRYPTFS",
        "nlk_cb_mutex-RDMA",
        "nlk_cb_mutex-CRYPTO",
        "nlk_cb_mutex-SMC",
        "nlk_cb_mutex-23",
        "nlk_cb_mutex-24",
        "nlk_cb_mutex-25",
        "nlk_cb_mutex-26",
        "nlk_cb_mutex-27",
        "nlk_cb_mutex-28",
        "nlk_cb_mutex-29",
        "nlk_cb_mutex-30",
        "nlk_cb_mutex-31",
        "nlk_cb_mutex-MAX_LINKS"
};

static int netlink_dump(struct sock *sk, bool lock_taken);

/* nl_table locking explained:
 * Lookup and traversal are protected with an RCU read-side lock. Insertion
 * and removal are protected with per bucket lock while using RCU list
 * modification primitives and may run in parallel to RCU protected lookups.
 * Destruction of the Netlink socket may only occur *after* nl_table_lock has
 * been acquired * either during or after the socket has been removed from
 * the list and after an RCU grace period.
 */
DEFINE_RWLOCK(nl_table_lock);
EXPORT_SYMBOL_GPL(nl_table_lock);
static atomic_t nl_table_users = ATOMIC_INIT(0);

#define nl_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&nl_table_lock));

static BLOCKING_NOTIFIER_HEAD(netlink_chain);


static const struct rhashtable_params netlink_rhashtable_params;

void do_trace_netlink_extack(const char *msg)
{
        trace_netlink_extack(msg);
}
EXPORT_SYMBOL(do_trace_netlink_extack);

static inline u32 netlink_group_mask(u32 group)
{
        if (group > 32)
                return 0;
        return group ? 1 << (group - 1) : 0;
}

static struct sk_buff *netlink_to_full_skb(const struct sk_buff *skb,
                                           gfp_t gfp_mask)
{
        unsigned int len = skb->len;
        struct sk_buff *new;

        new = alloc_skb(len, gfp_mask);
        if (new == NULL)
                return NULL;

        NETLINK_CB(new).portid = NETLINK_CB(skb).portid;
        NETLINK_CB(new).dst_group = NETLINK_CB(skb).dst_group;
        NETLINK_CB(new).creds = NETLINK_CB(skb).creds;

        skb_put_data(new, skb->data, len);
        return new;
}

static unsigned int netlink_tap_net_id;

struct netlink_tap_net {
        struct list_head netlink_tap_all;
        struct mutex netlink_tap_lock;
};

int netlink_add_tap(struct netlink_tap *nt)
{
        struct net *net = dev_net(nt->dev);
        struct netlink_tap_net *nn = net_generic(net, netlink_tap_net_id);

        if (unlikely(nt->dev->type != ARPHRD_NETLINK))
                return -EINVAL;

        mutex_lock(&nn->netlink_tap_lock);
        list_add_rcu(&nt->list, &nn->netlink_tap_all);
        mutex_unlock(&nn->netlink_tap_lock);

        __module_get(nt->module);

        return 0;
}
EXPORT_SYMBOL_GPL(netlink_add_tap);

static int __netlink_remove_tap(struct netlink_tap *nt)
{
        struct net *net = dev_net(nt->dev);
        struct netlink_tap_net *nn = net_generic(net, netlink_tap_net_id);
        bool found = false;
        struct netlink_tap *tmp;

        mutex_lock(&nn->netlink_tap_lock);

        list_for_each_entry(tmp, &nn->netlink_tap_all, list) {
                if (nt == tmp) {
                        list_del_rcu(&nt->list);
                        found = true;
                        goto out;
                }
        }

        pr_warn("__netlink_remove_tap: %p not found\n", nt);
out:
        mutex_unlock(&nn->netlink_tap_lock);

        if (found)
                module_put(nt->module);

        return found ? 0 : -ENODEV;
}

int netlink_remove_tap(struct netlink_tap *nt)
{
        int ret;

        ret = __netlink_remove_tap(nt);
        synchronize_net();

        return ret;
}
EXPORT_SYMBOL_GPL(netlink_remove_tap);

static __net_init int netlink_tap_init_net(struct net *net)
{
        struct netlink_tap_net *nn = net_generic(net, netlink_tap_net_id);

        INIT_LIST_HEAD(&nn->netlink_tap_all);
        mutex_init(&nn->netlink_tap_lock);
        return 0;
}

static struct pernet_operations netlink_tap_net_ops = {
        .init = netlink_tap_init_net,
        .id   = &netlink_tap_net_id,
        .size = sizeof(struct netlink_tap_net),
};

static bool netlink_filter_tap(const struct sk_buff *skb)
{
        struct sock *sk = skb->sk;

        /* We take the more conservative approach and
         * whitelist socket protocols that may pass.
         */
        switch (sk->sk_protocol) {
        case NETLINK_ROUTE:
        case NETLINK_USERSOCK:
        case NETLINK_SOCK_DIAG:
        case NETLINK_NFLOG:
        case NETLINK_XFRM:
        case NETLINK_FIB_LOOKUP:
        case NETLINK_NETFILTER:
        case NETLINK_GENERIC:
                return true;
        }

        return false;
}

static int __netlink_deliver_tap_skb(struct sk_buff *skb,
                                     struct net_device *dev)
{
        struct sk_buff *nskb;
        struct sock *sk = skb->sk;
        int ret = -ENOMEM;

        if (!net_eq(dev_net(dev), sock_net(sk)))
                return 0;

        dev_hold(dev);

        if (is_vmalloc_addr(skb->head))
                nskb = netlink_to_full_skb(skb, GFP_ATOMIC);
        else
                nskb = skb_clone(skb, GFP_ATOMIC);
        if (nskb) {
                nskb->dev = dev;
                nskb->protocol = htons((u16) sk->sk_protocol);
                nskb->pkt_type = netlink_is_kernel(sk) ?
                                 PACKET_KERNEL : PACKET_USER;
                skb_reset_network_header(nskb);
                ret = dev_queue_xmit(nskb);
                if (unlikely(ret > 0))
                        ret = net_xmit_errno(ret);
        }

        dev_put(dev);
        return ret;
}

static void __netlink_deliver_tap(struct sk_buff *skb, struct netlink_tap_net *nn)
{
        int ret;
        struct netlink_tap *tmp;

        if (!netlink_filter_tap(skb))
                return;

        list_for_each_entry_rcu(tmp, &nn->netlink_tap_all, list) {
                ret = __netlink_deliver_tap_skb(skb, tmp->dev);
                if (unlikely(ret))
                        break;
        }
}

static void netlink_deliver_tap(struct net *net, struct sk_buff *skb)
{
        struct netlink_tap_net *nn = net_generic(net, netlink_tap_net_id);

        rcu_read_lock();

        if (unlikely(!list_empty(&nn->netlink_tap_all)))
                __netlink_deliver_tap(skb, nn);

        rcu_read_unlock();
}

static void netlink_deliver_tap_kernel(struct sock *dst, struct sock *src,
                                       struct sk_buff *skb)
{
        if (!(netlink_is_kernel(dst) && netlink_is_kernel(src)))
                netlink_deliver_tap(sock_net(dst), skb);
}

static void netlink_overrun(struct sock *sk)
{
        if (!nlk_test_bit(RECV_NO_ENOBUFS, sk)) {
                if (!test_and_set_bit(NETLINK_S_CONGESTED,
                                      &nlk_sk(sk)->state)) {
                        WRITE_ONCE(sk->sk_err, ENOBUFS);
                        sk_error_report(sk);
                }
        }
        sk_drops_inc(sk);
}

static void netlink_rcv_wake(struct sock *sk)
{
        struct netlink_sock *nlk = nlk_sk(sk);

        if (skb_queue_empty_lockless(&sk->sk_receive_queue))
                clear_bit(NETLINK_S_CONGESTED, &nlk->state);
        if (!test_bit(NETLINK_S_CONGESTED, &nlk->state))
                wake_up_interruptible(&nlk->wait);
}

static void netlink_skb_destructor(struct sk_buff *skb)
{
        if (is_vmalloc_addr(skb->head)) {
                if (!skb->cloned ||
                    !atomic_dec_return(&(skb_shinfo(skb)->dataref)))
                        vfree_atomic(skb->head);

                skb->head = NULL;
        }
        if (skb->sk != NULL)
                sock_rfree(skb);
}

static void netlink_skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
{
        WARN_ON(skb->sk != NULL);
        skb->sk = sk;
        skb->destructor = netlink_skb_destructor;
        sk_mem_charge(sk, skb->truesize);
}

static void netlink_sock_destruct(struct sock *sk)
{
        skb_queue_purge(&sk->sk_receive_queue);

        if (!sock_flag(sk, SOCK_DEAD)) {
                printk(KERN_ERR "Freeing alive netlink socket %p\n", sk);
                return;
        }

        WARN_ON(atomic_read(&sk->sk_rmem_alloc));
        WARN_ON(refcount_read(&sk->sk_wmem_alloc));
        WARN_ON(nlk_sk(sk)->groups);
}

/* This lock without WQ_FLAG_EXCLUSIVE is good on UP and it is _very_ bad on
 * SMP. Look, when several writers sleep and reader wakes them up, all but one
 * immediately hit write lock and grab all the cpus. Exclusive sleep solves
 * this, _but_ remember, it adds useless work on UP machines.
 */

void netlink_table_grab(void)
        __acquires(nl_table_lock)
{
        might_sleep();

        write_lock_irq(&nl_table_lock);

        if (atomic_read(&nl_table_users)) {
                DECLARE_WAITQUEUE(wait, current);

                add_wait_queue_exclusive(&nl_table_wait, &wait);
                for (;;) {
                        set_current_state(TASK_UNINTERRUPTIBLE);
                        if (atomic_read(&nl_table_users) == 0)
                                break;
                        write_unlock_irq(&nl_table_lock);
                        schedule();
                        write_lock_irq(&nl_table_lock);
                }

                __set_current_state(TASK_RUNNING);
                remove_wait_queue(&nl_table_wait, &wait);
        }
}

void netlink_table_ungrab(void)
        __releases(nl_table_lock)
{
        write_unlock_irq(&nl_table_lock);
        wake_up(&nl_table_wait);
}

static inline void
netlink_lock_table(void)
{
        unsigned long flags;

        /* read_lock() synchronizes us to netlink_table_grab */

        read_lock_irqsave(&nl_table_lock, flags);
        atomic_inc(&nl_table_users);
        read_unlock_irqrestore(&nl_table_lock, flags);
}

static inline void
netlink_unlock_table(void)
{
        if (atomic_dec_and_test(&nl_table_users))
                wake_up(&nl_table_wait);
}

struct netlink_compare_arg
{
        possible_net_t pnet;
        u32 portid;
};

/* Doing sizeof directly may yield 4 extra bytes on 64-bit. */
#define netlink_compare_arg_len \
        (offsetof(struct netlink_compare_arg, portid) + sizeof(u32))

static inline int netlink_compare(struct rhashtable_compare_arg *arg,
                                  const void *ptr)
{
        const struct netlink_compare_arg *x = arg->key;
        const struct netlink_sock *nlk = ptr;

        return nlk->portid != x->portid ||
               !net_eq(sock_net(&nlk->sk), read_pnet(&x->pnet));
}

static void netlink_compare_arg_init(struct netlink_compare_arg *arg,
                                     struct net *net, u32 portid)
{
        memset(arg, 0, sizeof(*arg));
        write_pnet(&arg->pnet, net);
        arg->portid = portid;
}

static struct sock *__netlink_lookup(struct netlink_table *table, u32 portid,
                                     struct net *net)
{
        struct netlink_compare_arg arg;

        netlink_compare_arg_init(&arg, net, portid);
        return rhashtable_lookup_fast(&table->hash, &arg,
                                      netlink_rhashtable_params);
}

static int __netlink_insert(struct netlink_table *table, struct sock *sk)
{
        struct netlink_compare_arg arg;

        netlink_compare_arg_init(&arg, sock_net(sk), nlk_sk(sk)->portid);
        return rhashtable_lookup_insert_key(&table->hash, &arg,
                                            &nlk_sk(sk)->node,
                                            netlink_rhashtable_params);
}

static struct sock *netlink_lookup(struct net *net, int protocol, u32 portid)
{
        struct netlink_table *table = &nl_table[protocol];
        struct sock *sk;

        rcu_read_lock();
        sk = __netlink_lookup(table, portid, net);
        if (sk)
                sock_hold(sk);
        rcu_read_unlock();

        return sk;
}

static const struct proto_ops netlink_ops;

static void
netlink_update_listeners(struct sock *sk)
{
        struct netlink_table *tbl = &nl_table[sk->sk_protocol];
        unsigned long mask;
        unsigned int i;
        struct listeners *listeners;

        listeners = nl_deref_protected(tbl->listeners);
        if (!listeners)
                return;

        for (i = 0; i < NLGRPLONGS(tbl->groups); i++) {
                mask = 0;
                sk_for_each_bound(sk, &tbl->mc_list) {
                        if (i < NLGRPLONGS(nlk_sk(sk)->ngroups))
                                mask |= nlk_sk(sk)->groups[i];
                }
                listeners->masks[i] = mask;
        }
        /* this function is only called with the netlink table "grabbed", which
         * makes sure updates are visible before bind or setsockopt return. */
}

static int netlink_insert(struct sock *sk, u32 portid)
{
        struct netlink_table *table = &nl_table[sk->sk_protocol];
        int err;

        lock_sock(sk);

        err = nlk_sk(sk)->portid == portid ? 0 : -EBUSY;
        if (nlk_sk(sk)->bound)
                goto err;

        /* portid can be read locklessly from netlink_getname(). */
        WRITE_ONCE(nlk_sk(sk)->portid, portid);

        sock_hold(sk);

        err = __netlink_insert(table, sk);
        if (err) {
                /* In case the hashtable backend returns with -EBUSY
                 * from here, it must not escape to the caller.
                 */
                if (unlikely(err == -EBUSY))
                        err = -EOVERFLOW;
                if (err == -EEXIST)
                        err = -EADDRINUSE;
                sock_put(sk);
                goto err;
        }

        /* We need to ensure that the socket is hashed and visible. */
        smp_wmb();
        /* Paired with lockless reads from netlink_bind(),
         * netlink_connect() and netlink_sendmsg().
         */
        WRITE_ONCE(nlk_sk(sk)->bound, portid);

err:
        release_sock(sk);
        return err;
}

static void netlink_remove(struct sock *sk)
{
        struct netlink_table *table;

        table = &nl_table[sk->sk_protocol];
        if (!rhashtable_remove_fast(&table->hash, &nlk_sk(sk)->node,
                                    netlink_rhashtable_params)) {
                WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
                __sock_put(sk);
        }

        netlink_table_grab();
        if (nlk_sk(sk)->subscriptions) {
                __sk_del_bind_node(sk);
                netlink_update_listeners(sk);
        }
        if (sk->sk_protocol == NETLINK_GENERIC)
                atomic_inc(&genl_sk_destructing_cnt);
        netlink_table_ungrab();
}

static struct proto netlink_proto = {
        .name          = "NETLINK",
        .owner          = THIS_MODULE,
        .obj_size = sizeof(struct netlink_sock),
};

static int __netlink_create(struct net *net, struct socket *sock,
                            int protocol, int kern)
{
        struct sock *sk;
        struct netlink_sock *nlk;

        sock->ops = &netlink_ops;

        sk = sk_alloc(net, PF_NETLINK, GFP_KERNEL, &netlink_proto, kern);
        if (!sk)
                return -ENOMEM;

        sock_init_data(sock, sk);

        nlk = nlk_sk(sk);
        mutex_init(&nlk->nl_cb_mutex);
        lockdep_set_class_and_name(&nlk->nl_cb_mutex,
                                           nlk_cb_mutex_keys + protocol,
                                           nlk_cb_mutex_key_strings[protocol]);
        init_waitqueue_head(&nlk->wait);

        sk->sk_destruct = netlink_sock_destruct;
        sk->sk_protocol = protocol;
        return 0;
}

static int netlink_create(struct net *net, struct socket *sock, int protocol,
                          int kern)
{
        struct module *module = NULL;
        struct netlink_sock *nlk;
        int (*bind)(struct net *net, int group);
        void (*unbind)(struct net *net, int group);
        void (*release)(struct sock *sock, unsigned long *groups);
        int err = 0;

        sock->state = SS_UNCONNECTED;

        if (sock->type != SOCK_RAW && sock->type != SOCK_DGRAM)
                return -ESOCKTNOSUPPORT;

        if (protocol < 0 || protocol >= MAX_LINKS)
                return -EPROTONOSUPPORT;
        protocol = array_index_nospec(protocol, MAX_LINKS);

        netlink_lock_table();
#ifdef CONFIG_MODULES
        if (!nl_table[protocol].registered) {
                netlink_unlock_table();
                request_module("net-pf-%d-proto-%d", PF_NETLINK, protocol);
                netlink_lock_table();
        }
#endif
        if (nl_table[protocol].registered &&
            try_module_get(nl_table[protocol].module))
                module = nl_table[protocol].module;
        else
                err = -EPROTONOSUPPORT;
        bind = nl_table[protocol].bind;
        unbind = nl_table[protocol].unbind;
        release = nl_table[protocol].release;
        netlink_unlock_table();

        if (err < 0)
                goto out;

        err = __netlink_create(net, sock, protocol, kern);
        if (err < 0)
                goto out_module;

        sock_prot_inuse_add(net, &netlink_proto, 1);

        nlk = nlk_sk(sock->sk);
        nlk->module = module;
        nlk->netlink_bind = bind;
        nlk->netlink_unbind = unbind;
        nlk->netlink_release = release;
out:
        return err;

out_module:
        module_put(module);
        goto out;
}

static void deferred_put_nlk_sk(struct rcu_head *head)
{
        struct netlink_sock *nlk = container_of(head, struct netlink_sock, rcu);
        struct sock *sk = &nlk->sk;

        kfree(nlk->groups);
        nlk->groups = NULL;

        if (!refcount_dec_and_test(&sk->sk_refcnt))
                return;

        sk_free(sk);
}

static int netlink_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk;

        if (!sk)
                return 0;

        netlink_remove(sk);
        sock_orphan(sk);
        nlk = nlk_sk(sk);

        /*
         * OK. Socket is unlinked, any packets that arrive now
         * will be purged.
         */
        if (nlk->netlink_release)
                nlk->netlink_release(sk, nlk->groups);

        /* must not acquire netlink_table_lock in any way again before unbind
         * and notifying genetlink is done as otherwise it might deadlock
         */
        if (nlk->netlink_unbind) {
                int i;

                for (i = 0; i < nlk->ngroups; i++)
                        if (test_bit(i, nlk->groups))
                                nlk->netlink_unbind(sock_net(sk), i + 1);
        }
        if (sk->sk_protocol == NETLINK_GENERIC &&
            atomic_dec_return(&genl_sk_destructing_cnt) == 0)
                wake_up(&genl_sk_destructing_waitq);

        sock->sk = NULL;
        wake_up_interruptible_all(&nlk->wait);

        skb_queue_purge(&sk->sk_write_queue);

        if (nlk->portid && nlk->bound) {
                struct netlink_notify n = {
                                                .net = sock_net(sk),
                                                .protocol = sk->sk_protocol,
                                                .portid = nlk->portid,
                                          };
                blocking_notifier_call_chain(&netlink_chain,
                                NETLINK_URELEASE, &n);
        }

        /* Terminate any outstanding dump */
        if (nlk->cb_running) {
                if (nlk->cb.done)
                        nlk->cb.done(&nlk->cb);
                module_put(nlk->cb.module);
                kfree_skb(nlk->cb.skb);
                WRITE_ONCE(nlk->cb_running, false);
        }

        module_put(nlk->module);

        if (netlink_is_kernel(sk)) {
                netlink_table_grab();
                BUG_ON(nl_table[sk->sk_protocol].registered == 0);
                if (--nl_table[sk->sk_protocol].registered == 0) {
                        struct listeners *old;

                        old = nl_deref_protected(nl_table[sk->sk_protocol].listeners);
                        RCU_INIT_POINTER(nl_table[sk->sk_protocol].listeners, NULL);
                        kfree_rcu(old, rcu);
                        nl_table[sk->sk_protocol].module = NULL;
                        nl_table[sk->sk_protocol].bind = NULL;
                        nl_table[sk->sk_protocol].unbind = NULL;
                        nl_table[sk->sk_protocol].flags = 0;
                        nl_table[sk->sk_protocol].registered = 0;
                }
                netlink_table_ungrab();
        }

        sock_prot_inuse_add(sock_net(sk), &netlink_proto, -1);

        call_rcu(&nlk->rcu, deferred_put_nlk_sk);
        return 0;
}

static int netlink_autobind(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct net *net = sock_net(sk);
        struct netlink_table *table = &nl_table[sk->sk_protocol];
        s32 portid = task_tgid_vnr(current);
        int err;
        s32 rover = -4096;
        bool ok;

retry:
        cond_resched();
        rcu_read_lock();
        ok = !__netlink_lookup(table, portid, net);
        rcu_read_unlock();
        if (!ok) {
                /* Bind collision, search negative portid values. */
                if (rover == -4096)
                        /* rover will be in range [S32_MIN, -4097] */
                        rover = S32_MIN + get_random_u32_below(-4096 - S32_MIN);
                else if (rover >= -4096)
                        rover = -4097;
                portid = rover--;
                goto retry;
        }

        err = netlink_insert(sk, portid);
        if (err == -EADDRINUSE)
                goto retry;

        /* If 2 threads race to autobind, that is fine.  */
        if (err == -EBUSY)
                err = 0;

        return err;
}

/**
 * __netlink_ns_capable - General netlink message capability test
 * @nsp: NETLINK_CB of the socket buffer holding a netlink command from userspace.
 * @user_ns: The user namespace of the capability to use
 * @cap: The capability to use
 *
 * Test to see if the opener of the socket we received the message
 * from had when the netlink socket was created and the sender of the
 * message has the capability @cap in the user namespace @user_ns.
 */
bool __netlink_ns_capable(const struct netlink_skb_parms *nsp,
                        struct user_namespace *user_ns, int cap)
{
        return ((nsp->flags & NETLINK_SKB_DST) ||
                file_ns_capable(nsp->sk->sk_socket->file, user_ns, cap)) &&
                ns_capable(user_ns, cap);
}
EXPORT_SYMBOL(__netlink_ns_capable);

/**
 * netlink_ns_capable - General netlink message capability test
 * @skb: socket buffer holding a netlink command from userspace
 * @user_ns: The user namespace of the capability to use
 * @cap: The capability to use
 *
 * Test to see if the opener of the socket we received the message
 * from had when the netlink socket was created and the sender of the
 * message has the capability @cap in the user namespace @user_ns.
 */
bool netlink_ns_capable(const struct sk_buff *skb,
                        struct user_namespace *user_ns, int cap)
{
        return __netlink_ns_capable(&NETLINK_CB(skb), user_ns, cap);
}
EXPORT_SYMBOL(netlink_ns_capable);

/**
 * netlink_capable - Netlink global message capability test
 * @skb: socket buffer holding a netlink command from userspace
 * @cap: The capability to use
 *
 * Test to see if the opener of the socket we received the message
 * from had when the netlink socket was created and the sender of the
 * message has the capability @cap in all user namespaces.
 */
bool netlink_capable(const struct sk_buff *skb, int cap)
{
        return netlink_ns_capable(skb, &init_user_ns, cap);
}
EXPORT_SYMBOL(netlink_capable);

/**
 * netlink_net_capable - Netlink network namespace message capability test
 * @skb: socket buffer holding a netlink command from userspace
 * @cap: The capability to use
 *
 * Test to see if the opener of the socket we received the message
 * from had when the netlink socket was created and the sender of the
 * message has the capability @cap over the network namespace of
 * the socket we received the message from.
 */
bool netlink_net_capable(const struct sk_buff *skb, int cap)
{
        return netlink_ns_capable(skb, sock_net(skb->sk)->user_ns, cap);
}
EXPORT_SYMBOL(netlink_net_capable);

static inline int netlink_allowed(const struct socket *sock, unsigned int flag)
{
        return (nl_table[sock->sk->sk_protocol].flags & flag) ||
                ns_capable(sock_net(sock->sk)->user_ns, CAP_NET_ADMIN);
}

static void
netlink_update_subscriptions(struct sock *sk, unsigned int subscriptions)
{
        struct netlink_sock *nlk = nlk_sk(sk);

        if (nlk->subscriptions && !subscriptions)
                __sk_del_bind_node(sk);
        else if (!nlk->subscriptions && subscriptions)
                sk_add_bind_node(sk, &nl_table[sk->sk_protocol].mc_list);
        nlk->subscriptions = subscriptions;
}

static int netlink_realloc_groups(struct sock *sk)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        unsigned int groups;
        unsigned long *new_groups;
        int err = 0;

        netlink_table_grab();

        groups = nl_table[sk->sk_protocol].groups;
        if (!nl_table[sk->sk_protocol].registered) {
                err = -ENOENT;
                goto out_unlock;
        }

        if (nlk->ngroups >= groups)
                goto out_unlock;

        new_groups = krealloc(nlk->groups, NLGRPSZ(groups), GFP_ATOMIC);
        if (new_groups == NULL) {
                err = -ENOMEM;
                goto out_unlock;
        }
        memset((char *)new_groups + NLGRPSZ(nlk->ngroups), 0,
               NLGRPSZ(groups) - NLGRPSZ(nlk->ngroups));

        nlk->groups = new_groups;
        nlk->ngroups = groups;
 out_unlock:
        netlink_table_ungrab();
        return err;
}

static void netlink_undo_bind(int group, long unsigned int groups,
                              struct sock *sk)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        int undo;

        if (!nlk->netlink_unbind)
                return;

        for (undo = 0; undo < group; undo++)
                if (test_bit(undo, &groups))
                        nlk->netlink_unbind(sock_net(sk), undo + 1);
}

static int netlink_bind(struct socket *sock, struct sockaddr *addr,
                        int addr_len)
{
        struct sock *sk = sock->sk;
        struct net *net = sock_net(sk);
        struct netlink_sock *nlk = nlk_sk(sk);
        struct sockaddr_nl *nladdr = (struct sockaddr_nl *)addr;
        int err = 0;
        unsigned long groups;
        bool bound;

        if (addr_len < sizeof(struct sockaddr_nl))
                return -EINVAL;

        if (nladdr->nl_family != AF_NETLINK)
                return -EINVAL;
        groups = nladdr->nl_groups;

        /* Only superuser is allowed to listen multicasts */
        if (groups) {
                if (!netlink_allowed(sock, NL_CFG_F_NONROOT_RECV))
                        return -EPERM;
                err = netlink_realloc_groups(sk);
                if (err)
                        return err;
        }

        if (nlk->ngroups < BITS_PER_LONG)
                groups &= (1UL << nlk->ngroups) - 1;

        /* Paired with WRITE_ONCE() in netlink_insert() */
        bound = READ_ONCE(nlk->bound);
        if (bound) {
                /* Ensure nlk->portid is up-to-date. */
                smp_rmb();

                if (nladdr->nl_pid != nlk->portid)
                        return -EINVAL;
        }

        if (nlk->netlink_bind && groups) {
                int group;

                /* nl_groups is a u32, so cap the maximum groups we can bind */
                for (group = 0; group < BITS_PER_TYPE(u32); group++) {
                        if (!test_bit(group, &groups))
                                continue;
                        err = nlk->netlink_bind(net, group + 1);
                        if (!err)
                                continue;
                        netlink_undo_bind(group, groups, sk);
                        return err;
                }
        }

        /* No need for barriers here as we return to user-space without
         * using any of the bound attributes.
         */
        netlink_lock_table();
        if (!bound) {
                err = nladdr->nl_pid ?
                        netlink_insert(sk, nladdr->nl_pid) :
                        netlink_autobind(sock);
                if (err) {
                        netlink_undo_bind(BITS_PER_TYPE(u32), groups, sk);
                        goto unlock;
                }
        }

        if (!groups && (nlk->groups == NULL || !(u32)nlk->groups[0]))
                goto unlock;
        netlink_unlock_table();

        netlink_table_grab();
        netlink_update_subscriptions(sk, nlk->subscriptions +
                                         hweight32(groups) -
                                         hweight32(nlk->groups[0]));
        nlk->groups[0] = (nlk->groups[0] & ~0xffffffffUL) | groups;
        netlink_update_listeners(sk);
        netlink_table_ungrab();

        return 0;

unlock:
        netlink_unlock_table();
        return err;
}

static int netlink_connect(struct socket *sock, struct sockaddr *addr,
                           int alen, int flags)
{
        int err = 0;
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        struct sockaddr_nl *nladdr = (struct sockaddr_nl *)addr;

        if (alen < sizeof(addr->sa_family))
                return -EINVAL;

        if (addr->sa_family == AF_UNSPEC) {
                /* paired with READ_ONCE() in netlink_getsockbyportid() */
                WRITE_ONCE(sk->sk_state, NETLINK_UNCONNECTED);
                /* dst_portid and dst_group can be read locklessly */
                WRITE_ONCE(nlk->dst_portid, 0);
                WRITE_ONCE(nlk->dst_group, 0);
                return 0;
        }
        if (addr->sa_family != AF_NETLINK)
                return -EINVAL;

        if (alen < sizeof(struct sockaddr_nl))
                return -EINVAL;

        if ((nladdr->nl_groups || nladdr->nl_pid) &&
            !netlink_allowed(sock, NL_CFG_F_NONROOT_SEND))
                return -EPERM;

        /* No need for barriers here as we return to user-space without
         * using any of the bound attributes.
         * Paired with WRITE_ONCE() in netlink_insert().
         */
        if (!READ_ONCE(nlk->bound))
                err = netlink_autobind(sock);

        if (err == 0) {
                /* paired with READ_ONCE() in netlink_getsockbyportid() */
                WRITE_ONCE(sk->sk_state, NETLINK_CONNECTED);
                /* dst_portid and dst_group can be read locklessly */
                WRITE_ONCE(nlk->dst_portid, nladdr->nl_pid);
                WRITE_ONCE(nlk->dst_group, ffs(nladdr->nl_groups));
        }

        return err;
}

static int netlink_getname(struct socket *sock, struct sockaddr *addr,
                           int peer)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        DECLARE_SOCKADDR(struct sockaddr_nl *, nladdr, addr);

        nladdr->nl_family = AF_NETLINK;
        nladdr->nl_pad = 0;

        if (peer) {
                /* Paired with WRITE_ONCE() in netlink_connect() */
                nladdr->nl_pid = READ_ONCE(nlk->dst_portid);
                nladdr->nl_groups = netlink_group_mask(READ_ONCE(nlk->dst_group));
        } else {
                /* Paired with WRITE_ONCE() in netlink_insert() */
                nladdr->nl_pid = READ_ONCE(nlk->portid);
                netlink_lock_table();
                nladdr->nl_groups = nlk->groups ? nlk->groups[0] : 0;
                netlink_unlock_table();
        }
        return sizeof(*nladdr);
}

static int netlink_ioctl(struct socket *sock, unsigned int cmd,
                         unsigned long arg)
{
        /* try to hand this ioctl down to the NIC drivers.
         */
        return -ENOIOCTLCMD;
}

static struct sock *netlink_getsockbyportid(struct sock *ssk, u32 portid)
{
        struct sock *sock;
        struct netlink_sock *nlk;

        sock = netlink_lookup(sock_net(ssk), ssk->sk_protocol, portid);
        if (!sock)
                return ERR_PTR(-ECONNREFUSED);

        /* Don't bother queuing skb if kernel socket has no input function */
        nlk = nlk_sk(sock);
        /* dst_portid and sk_state can be changed in netlink_connect() */
        if (READ_ONCE(sock->sk_state) == NETLINK_CONNECTED &&
            READ_ONCE(nlk->dst_portid) != nlk_sk(ssk)->portid) {
                sock_put(sock);
                return ERR_PTR(-ECONNREFUSED);
        }
        return sock;
}

struct sock *netlink_getsockbyfd(int fd)
{
        CLASS(fd, f)(fd);
        struct inode *inode;
        struct sock *sock;

        if (fd_empty(f))
                return ERR_PTR(-EBADF);

        inode = file_inode(fd_file(f));
        if (!S_ISSOCK(inode->i_mode))
                return ERR_PTR(-ENOTSOCK);

        sock = SOCKET_I(inode)->sk;
        if (sock->sk_family != AF_NETLINK)
                return ERR_PTR(-EINVAL);

        sock_hold(sock);
        return sock;
}

struct sk_buff *netlink_alloc_large_skb(unsigned int size, int broadcast)
{
        size_t head_size = SKB_HEAD_ALIGN(size);
        struct sk_buff *skb;
        void *data;

        if (head_size <= PAGE_SIZE || broadcast)
                return alloc_skb(size, GFP_KERNEL);

        data = kvmalloc(head_size, GFP_KERNEL);
        if (!data)
                return NULL;

        skb = __build_skb(data, head_size);
        if (!skb)
                kvfree(data);
        else if (is_vmalloc_addr(data))
                skb->destructor = netlink_skb_destructor;

        return skb;
}

/*
 * Attach a skb to a netlink socket.
 * The caller must hold a reference to the destination socket. On error, the
 * reference is dropped. The skb is not send to the destination, just all
 * all error checks are performed and memory in the queue is reserved.
 * Return values:
 * < 0: error. skb freed, reference to sock dropped.
 * 0: continue
 * 1: repeat lookup - reference dropped while waiting for socket memory.
 */
int netlink_attachskb(struct sock *sk, struct sk_buff *skb,
                      long *timeo, struct sock *ssk)
{
        DECLARE_WAITQUEUE(wait, current);
        struct netlink_sock *nlk;
        unsigned int rmem;

        nlk = nlk_sk(sk);
        rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc);

        if ((rmem == skb->truesize || rmem <= READ_ONCE(sk->sk_rcvbuf)) &&
            !test_bit(NETLINK_S_CONGESTED, &nlk->state)) {
                netlink_skb_set_owner_r(skb, sk);
                return 0;
        }

        atomic_sub(skb->truesize, &sk->sk_rmem_alloc);

        if (!*timeo) {
                if (!ssk || netlink_is_kernel(ssk))
                        netlink_overrun(sk);
                sock_put(sk);
                kfree_skb(skb);
                return -EAGAIN;
        }

        __set_current_state(TASK_INTERRUPTIBLE);
        add_wait_queue(&nlk->wait, &wait);
        rmem = atomic_read(&sk->sk_rmem_alloc);

        if (((rmem && rmem + skb->truesize > READ_ONCE(sk->sk_rcvbuf)) ||
             test_bit(NETLINK_S_CONGESTED, &nlk->state)) &&
            !sock_flag(sk, SOCK_DEAD))
                *timeo = schedule_timeout(*timeo);

        __set_current_state(TASK_RUNNING);
        remove_wait_queue(&nlk->wait, &wait);
        sock_put(sk);

        if (signal_pending(current)) {
                kfree_skb(skb);
                return sock_intr_errno(*timeo);
        }

        return 1;
}

static int __netlink_sendskb(struct sock *sk, struct sk_buff *skb)
{
        int len = skb->len;

        netlink_deliver_tap(sock_net(sk), skb);

        skb_queue_tail(&sk->sk_receive_queue, skb);
        sk->sk_data_ready(sk);
        return len;
}

int netlink_sendskb(struct sock *sk, struct sk_buff *skb)
{
        int len = __netlink_sendskb(sk, skb);

        sock_put(sk);
        return len;
}

void netlink_detachskb(struct sock *sk, struct sk_buff *skb)
{
        kfree_skb(skb);
        sock_put(sk);
}

static struct sk_buff *netlink_trim(struct sk_buff *skb, gfp_t allocation)
{
        int delta;

        skb_assert_len(skb);
        WARN_ON(skb->sk != NULL);
        delta = skb->end - skb->tail;
        if (is_vmalloc_addr(skb->head) || delta * 2 < skb->truesize)
                return skb;

        if (skb_shared(skb)) {
                struct sk_buff *nskb = skb_clone(skb, allocation);
                if (!nskb)
                        return skb;
                consume_skb(skb);
                skb = nskb;
        }

        pskb_expand_head(skb, 0, -delta,
                         (allocation & ~__GFP_DIRECT_RECLAIM) |
                         __GFP_NOWARN | __GFP_NORETRY);
        return skb;
}

static int netlink_unicast_kernel(struct sock *sk, struct sk_buff *skb,
                                  struct sock *ssk)
{
        int ret;
        struct netlink_sock *nlk = nlk_sk(sk);

        ret = -ECONNREFUSED;
        if (nlk->netlink_rcv != NULL) {
                ret = skb->len;
                atomic_add(skb->truesize, &sk->sk_rmem_alloc);
                netlink_skb_set_owner_r(skb, sk);
                NETLINK_CB(skb).sk = ssk;
                netlink_deliver_tap_kernel(sk, ssk, skb);
                nlk->netlink_rcv(skb);
                consume_skb(skb);
        } else {
                kfree_skb(skb);
        }
        sock_put(sk);
        return ret;
}

int netlink_unicast(struct sock *ssk, struct sk_buff *skb,
                    u32 portid, int nonblock)
{
        struct sock *sk;
        int err;
        long timeo;

        skb = netlink_trim(skb, gfp_any());

        timeo = sock_sndtimeo(ssk, nonblock);
retry:
        sk = netlink_getsockbyportid(ssk, portid);
        if (IS_ERR(sk)) {
                kfree_skb(skb);
                return PTR_ERR(sk);
        }
        if (netlink_is_kernel(sk))
                return netlink_unicast_kernel(sk, skb, ssk);

        if (sk_filter(sk, skb)) {
                err = skb->len;
                kfree_skb(skb);
                sock_put(sk);
                return err;
        }

        err = netlink_attachskb(sk, skb, &timeo, ssk);
        if (err == 1)
                goto retry;
        if (err)
                return err;

        return netlink_sendskb(sk, skb);
}
EXPORT_SYMBOL(netlink_unicast);

int netlink_has_listeners(struct sock *sk, unsigned int group)
{
        int res = 0;
        struct listeners *listeners;

        BUG_ON(!netlink_is_kernel(sk));

        rcu_read_lock();
        listeners = rcu_dereference(nl_table[sk->sk_protocol].listeners);

        if (listeners && group - 1 < nl_table[sk->sk_protocol].groups)
                res = test_bit(group - 1, listeners->masks);

        rcu_read_unlock();

        return res;
}
EXPORT_SYMBOL_GPL(netlink_has_listeners);

bool netlink_strict_get_check(struct sk_buff *skb)
{
        return nlk_test_bit(STRICT_CHK, NETLINK_CB(skb).sk);
}
EXPORT_SYMBOL_GPL(netlink_strict_get_check);

static int netlink_broadcast_deliver(struct sock *sk, struct sk_buff *skb)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        unsigned int rmem, rcvbuf;

        rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc);
        rcvbuf = READ_ONCE(sk->sk_rcvbuf);

        if ((rmem == skb->truesize || rmem <= rcvbuf) &&
            !test_bit(NETLINK_S_CONGESTED, &nlk->state)) {
                netlink_skb_set_owner_r(skb, sk);
                __netlink_sendskb(sk, skb);
                return rmem > (rcvbuf >> 1);
        }

        atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
        return -1;
}

struct netlink_broadcast_data {
        struct sock *exclude_sk;
        struct net *net;
        u32 portid;
        u32 group;
        int failure;
        int delivery_failure;
        int congested;
        int delivered;
        gfp_t allocation;
        struct sk_buff *skb, *skb2;
        int (*tx_filter)(struct sock *dsk, struct sk_buff *skb, void *data);
        void *tx_data;
};

static void do_one_broadcast(struct sock *sk,
                                    struct netlink_broadcast_data *p)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        int val;

        if (p->exclude_sk == sk)
                return;

        if (nlk->portid == p->portid || p->group - 1 >= nlk->ngroups ||
            !test_bit(p->group - 1, nlk->groups))
                return;

        if (!net_eq(sock_net(sk), p->net)) {
                if (!nlk_test_bit(LISTEN_ALL_NSID, sk))
                        return;

                if (!peernet_has_id(sock_net(sk), p->net))
                        return;

                if (!file_ns_capable(sk->sk_socket->file, p->net->user_ns,
                                     CAP_NET_BROADCAST))
                        return;
        }

        if (p->failure) {
                netlink_overrun(sk);
                return;
        }

        sock_hold(sk);
        if (p->skb2 == NULL) {
                if (skb_shared(p->skb)) {
                        p->skb2 = skb_clone(p->skb, p->allocation);
                } else {
                        p->skb2 = skb_get(p->skb);
                        /*
                         * skb ownership may have been set when
                         * delivered to a previous socket.
                         */
                        skb_orphan(p->skb2);
                }
        }
        if (p->skb2 == NULL) {
                netlink_overrun(sk);
                /* Clone failed. Notify ALL listeners. */
                p->failure = 1;
                if (nlk_test_bit(BROADCAST_SEND_ERROR, sk))
                        p->delivery_failure = 1;
                goto out;
        }

        if (p->tx_filter && p->tx_filter(sk, p->skb2, p->tx_data)) {
                kfree_skb(p->skb2);
                p->skb2 = NULL;
                goto out;
        }

        if (sk_filter(sk, p->skb2)) {
                kfree_skb(p->skb2);
                p->skb2 = NULL;
                goto out;
        }
        NETLINK_CB(p->skb2).nsid = peernet2id(sock_net(sk), p->net);
        if (NETLINK_CB(p->skb2).nsid != NETNSA_NSID_NOT_ASSIGNED)
                NETLINK_CB(p->skb2).nsid_is_set = true;
        val = netlink_broadcast_deliver(sk, p->skb2);
        if (val < 0) {
                netlink_overrun(sk);
                if (nlk_test_bit(BROADCAST_SEND_ERROR, sk))
                        p->delivery_failure = 1;
        } else {
                p->congested |= val;
                p->delivered = 1;
                p->skb2 = NULL;
        }
out:
        sock_put(sk);
}

int netlink_broadcast_filtered(struct sock *ssk, struct sk_buff *skb,
                               u32 portid,
                               u32 group, gfp_t allocation,
                               netlink_filter_fn filter,
                               void *filter_data)
{
        struct net *net = sock_net(ssk);
        struct netlink_broadcast_data info;
        struct sock *sk;

        skb = netlink_trim(skb, allocation);

        info.exclude_sk = ssk;
        info.net = net;
        info.portid = portid;
        info.group = group;
        info.failure = 0;
        info.delivery_failure = 0;
        info.congested = 0;
        info.delivered = 0;
        info.allocation = allocation;
        info.skb = skb;
        info.skb2 = NULL;
        info.tx_filter = filter;
        info.tx_data = filter_data;

        /* While we sleep in clone, do not allow to change socket list */

        netlink_lock_table();

        sk_for_each_bound(sk, &nl_table[ssk->sk_protocol].mc_list)
                do_one_broadcast(sk, &info);

        consume_skb(skb);

        netlink_unlock_table();

        if (info.delivery_failure) {
                kfree_skb(info.skb2);
                return -ENOBUFS;
        }
        consume_skb(info.skb2);

        if (info.delivered) {
                if (info.congested && gfpflags_allow_blocking(allocation))
                        yield();
                return 0;
        }
        return -ESRCH;
}
EXPORT_SYMBOL(netlink_broadcast_filtered);

int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, u32 portid,
                      u32 group, gfp_t allocation)
{
        return netlink_broadcast_filtered(ssk, skb, portid, group, allocation,
                                          NULL, NULL);
}
EXPORT_SYMBOL(netlink_broadcast);

struct netlink_set_err_data {
        struct sock *exclude_sk;
        u32 portid;
        u32 group;
        int code;
};

static int do_one_set_err(struct sock *sk, struct netlink_set_err_data *p)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        int ret = 0;

        if (sk == p->exclude_sk)
                goto out;

        if (!net_eq(sock_net(sk), sock_net(p->exclude_sk)))
                goto out;

        if (nlk->portid == p->portid || p->group - 1 >= nlk->ngroups ||
            !test_bit(p->group - 1, nlk->groups))
                goto out;

        if (p->code == ENOBUFS && nlk_test_bit(RECV_NO_ENOBUFS, sk)) {
                ret = 1;
                goto out;
        }

        WRITE_ONCE(sk->sk_err, p->code);
        sk_error_report(sk);
out:
        return ret;
}

/**
 * netlink_set_err - report error to broadcast listeners
 * @ssk: the kernel netlink socket, as returned by netlink_kernel_create()
 * @portid: the PORTID of a process that we want to skip (if any)
 * @group: the broadcast group that will notice the error
 * @code: error code, must be negative (as usual in kernelspace)
 *
 * This function returns the number of broadcast listeners that have set the
 * NETLINK_NO_ENOBUFS socket option.
 */
int netlink_set_err(struct sock *ssk, u32 portid, u32 group, int code)
{
        struct netlink_set_err_data info;
        unsigned long flags;
        struct sock *sk;
        int ret = 0;

        info.exclude_sk = ssk;
        info.portid = portid;
        info.group = group;
        /* sk->sk_err wants a positive error value */
        info.code = -code;

        read_lock_irqsave(&nl_table_lock, flags);

        sk_for_each_bound(sk, &nl_table[ssk->sk_protocol].mc_list)
                ret += do_one_set_err(sk, &info);

        read_unlock_irqrestore(&nl_table_lock, flags);
        return ret;
}
EXPORT_SYMBOL(netlink_set_err);

/* must be called with netlink table grabbed */
static void netlink_update_socket_mc(struct netlink_sock *nlk,
                                     unsigned int group,
                                     int is_new)
{
        int old, new = !!is_new, subscriptions;

        old = test_bit(group - 1, nlk->groups);
        subscriptions = nlk->subscriptions - old + new;
        __assign_bit(group - 1, nlk->groups, new);
        netlink_update_subscriptions(&nlk->sk, subscriptions);
        netlink_update_listeners(&nlk->sk);
}

static int netlink_setsockopt(struct socket *sock, int level, int optname,
                              sockptr_t optval, unsigned int optlen)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        unsigned int val = 0;
        int nr = -1;

        if (level != SOL_NETLINK)
                return -ENOPROTOOPT;

        if (optlen >= sizeof(int) &&
            copy_from_sockptr(&val, optval, sizeof(val)))
                return -EFAULT;

        switch (optname) {
        case NETLINK_PKTINFO:
                nr = NETLINK_F_RECV_PKTINFO;
                break;
        case NETLINK_ADD_MEMBERSHIP:
        case NETLINK_DROP_MEMBERSHIP: {
                int err;

                if (!netlink_allowed(sock, NL_CFG_F_NONROOT_RECV))
                        return -EPERM;
                err = netlink_realloc_groups(sk);
                if (err)
                        return err;
                if (!val || val - 1 >= nlk->ngroups)
                        return -EINVAL;
                if (optname == NETLINK_ADD_MEMBERSHIP && nlk->netlink_bind) {
                        err = nlk->netlink_bind(sock_net(sk), val);
                        if (err)
                                return err;
                }
                netlink_table_grab();
                netlink_update_socket_mc(nlk, val,
                                         optname == NETLINK_ADD_MEMBERSHIP);
                netlink_table_ungrab();
                if (optname == NETLINK_DROP_MEMBERSHIP && nlk->netlink_unbind)
                        nlk->netlink_unbind(sock_net(sk), val);

                break;
        }
        case NETLINK_BROADCAST_ERROR:
                nr = NETLINK_F_BROADCAST_SEND_ERROR;
                break;
        case NETLINK_NO_ENOBUFS:
                assign_bit(NETLINK_F_RECV_NO_ENOBUFS, &nlk->flags, val);
                if (val) {
                        clear_bit(NETLINK_S_CONGESTED, &nlk->state);
                        wake_up_interruptible(&nlk->wait);
                }
                break;
        case NETLINK_LISTEN_ALL_NSID:
                if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_BROADCAST))
                        return -EPERM;
                nr = NETLINK_F_LISTEN_ALL_NSID;
                break;
        case NETLINK_CAP_ACK:
                nr = NETLINK_F_CAP_ACK;
                break;
        case NETLINK_EXT_ACK:
                nr = NETLINK_F_EXT_ACK;
                break;
        case NETLINK_GET_STRICT_CHK:
                nr = NETLINK_F_STRICT_CHK;
                break;
        default:
                return -ENOPROTOOPT;
        }
        if (nr >= 0)
                assign_bit(nr, &nlk->flags, val);
        return 0;
}

static int netlink_getsockopt(struct socket *sock, int level, int optname,
                              char __user *optval, int __user *optlen)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        unsigned int flag;
        int len, val;

        if (level != SOL_NETLINK)
                return -ENOPROTOOPT;

        if (get_user(len, optlen))
                return -EFAULT;
        if (len < 0)
                return -EINVAL;

        switch (optname) {
        case NETLINK_PKTINFO:
                flag = NETLINK_F_RECV_PKTINFO;
                break;
        case NETLINK_BROADCAST_ERROR:
                flag = NETLINK_F_BROADCAST_SEND_ERROR;
                break;
        case NETLINK_NO_ENOBUFS:
                flag = NETLINK_F_RECV_NO_ENOBUFS;
                break;
        case NETLINK_LIST_MEMBERSHIPS: {
                int pos, idx, shift, err = 0;

                netlink_lock_table();
                for (pos = 0; pos * 8 < nlk->ngroups; pos += sizeof(u32)) {
                        if (len - pos < sizeof(u32))
                                break;

                        idx = pos / sizeof(unsigned long);
                        shift = (pos % sizeof(unsigned long)) * 8;
                        if (put_user((u32)(nlk->groups[idx] >> shift),
                                     (u32 __user *)(optval + pos))) {
                                err = -EFAULT;
                                break;
                        }
                }
                if (put_user(ALIGN(BITS_TO_BYTES(nlk->ngroups), sizeof(u32)), optlen))
                        err = -EFAULT;
                netlink_unlock_table();
                return err;
        }
        case NETLINK_LISTEN_ALL_NSID:
                flag = NETLINK_F_LISTEN_ALL_NSID;
                break;
        case NETLINK_CAP_ACK:
                flag = NETLINK_F_CAP_ACK;
                break;
        case NETLINK_EXT_ACK:
                flag = NETLINK_F_EXT_ACK;
                break;
        case NETLINK_GET_STRICT_CHK:
                flag = NETLINK_F_STRICT_CHK;
                break;
        default:
                return -ENOPROTOOPT;
        }

        if (len < sizeof(int))
                return -EINVAL;

        len = sizeof(int);
        val = test_bit(flag, &nlk->flags);

        if (put_user(len, optlen) ||
            copy_to_user(optval, &val, len))
                return -EFAULT;

        return 0;
}

static void netlink_cmsg_recv_pktinfo(struct msghdr *msg, struct sk_buff *skb)
{
        struct nl_pktinfo info;

        info.group = NETLINK_CB(skb).dst_group;
        put_cmsg(msg, SOL_NETLINK, NETLINK_PKTINFO, sizeof(info), &info);
}

static void netlink_cmsg_listen_all_nsid(struct sock *sk, struct msghdr *msg,
                                         struct sk_buff *skb)
{
        if (!NETLINK_CB(skb).nsid_is_set)
                return;

        put_cmsg(msg, SOL_NETLINK, NETLINK_LISTEN_ALL_NSID, sizeof(int),
                 &NETLINK_CB(skb).nsid);
}

static int netlink_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
{
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        DECLARE_SOCKADDR(struct sockaddr_nl *, addr, msg->msg_name);
        u32 dst_portid;
        u32 dst_group;
        struct sk_buff *skb;
        int err;
        struct scm_cookie scm;
        u32 netlink_skb_flags = 0;

        if (msg->msg_flags & MSG_OOB)
                return -EOPNOTSUPP;

        if (len == 0) {
                pr_warn_once("Zero length message leads to an empty skb\n");
                return -ENODATA;
        }

        err = scm_send(sock, msg, &scm, true);
        if (err < 0)
                return err;

        if (msg->msg_namelen) {
                err = -EINVAL;
                if (msg->msg_namelen < sizeof(struct sockaddr_nl))
                        goto out;
                if (addr->nl_family != AF_NETLINK)
                        goto out;
                dst_portid = addr->nl_pid;
                dst_group = ffs(addr->nl_groups);
                err =  -EPERM;
                if ((dst_group || dst_portid) &&
                    !netlink_allowed(sock, NL_CFG_F_NONROOT_SEND))
                        goto out;
                netlink_skb_flags |= NETLINK_SKB_DST;
        } else {
                /* Paired with WRITE_ONCE() in netlink_connect() */
                dst_portid = READ_ONCE(nlk->dst_portid);
                dst_group = READ_ONCE(nlk->dst_group);
        }

        /* Paired with WRITE_ONCE() in netlink_insert() */
        if (!READ_ONCE(nlk->bound)) {
                err = netlink_autobind(sock);
                if (err)
                        goto out;
        } else {
                /* Ensure nlk is hashed and visible. */
                smp_rmb();
        }

        err = -EMSGSIZE;
        if (len > sk->sk_sndbuf - 32)
                goto out;
        err = -ENOBUFS;
        skb = netlink_alloc_large_skb(len, dst_group);
        if (skb == NULL)
                goto out;

        NETLINK_CB(skb).portid        = nlk->portid;
        NETLINK_CB(skb).dst_group = dst_group;
        NETLINK_CB(skb).creds        = scm.creds;
        NETLINK_CB(skb).flags        = netlink_skb_flags;

        err = -EFAULT;
        if (memcpy_from_msg(skb_put(skb, len), msg, len)) {
                kfree_skb(skb);
                goto out;
        }

        err = security_netlink_send(sk, skb);
        if (err) {
                kfree_skb(skb);
                goto out;
        }

        if (dst_group) {
                refcount_inc(&skb->users);
                netlink_broadcast(sk, skb, dst_portid, dst_group, GFP_KERNEL);
        }
        err = netlink_unicast(sk, skb, dst_portid, msg->msg_flags & MSG_DONTWAIT);

out:
        scm_destroy(&scm);
        return err;
}

static int netlink_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
                           int flags)
{
        struct scm_cookie scm;
        struct sock *sk = sock->sk;
        struct netlink_sock *nlk = nlk_sk(sk);
        size_t copied, max_recvmsg_len;
        struct sk_buff *skb, *data_skb;
        int err, ret;

        if (flags & MSG_OOB)
                return -EOPNOTSUPP;

        copied = 0;

        skb = skb_recv_datagram(sk, flags, &err);
        if (skb == NULL)
                goto out;

        data_skb = skb;

#ifdef CONFIG_COMPAT_NETLINK_MESSAGES
        if (unlikely(skb_shinfo(skb)->frag_list)) {
                /*
                 * If this skb has a frag_list, then here that means that we
                 * will have to use the frag_list skb's data for compat tasks
                 * and the regular skb's data for normal (non-compat) tasks.
                 *
                 * If we need to send the compat skb, assign it to the
                 * 'data_skb' variable so that it will be used below for data
                 * copying. We keep 'skb' for everything else, including
                 * freeing both later.
                 */
                if (flags & MSG_CMSG_COMPAT)
                        data_skb = skb_shinfo(skb)->frag_list;
        }
#endif

        /* Record the max length of recvmsg() calls for future allocations */
        max_recvmsg_len = max(READ_ONCE(nlk->max_recvmsg_len), len);
        max_recvmsg_len = min_t(size_t, max_recvmsg_len,
                                SKB_WITH_OVERHEAD(32768));
        WRITE_ONCE(nlk->max_recvmsg_len, max_recvmsg_len);

        copied = data_skb->len;
        if (len < copied) {
                msg->msg_flags |= MSG_TRUNC;
                copied = len;
        }

        err = skb_copy_datagram_msg(data_skb, 0, msg, copied);

        if (msg->msg_name) {
                DECLARE_SOCKADDR(struct sockaddr_nl *, addr, msg->msg_name);
                addr->nl_family = AF_NETLINK;
                addr->nl_pad    = 0;
                addr->nl_pid        = NETLINK_CB(skb).portid;
                addr->nl_groups        = netlink_group_mask(NETLINK_CB(skb).dst_group);
                msg->msg_namelen = sizeof(*addr);
        }

        if (nlk_test_bit(RECV_PKTINFO, sk))
                netlink_cmsg_recv_pktinfo(msg, skb);
        if (nlk_test_bit(LISTEN_ALL_NSID, sk))
                netlink_cmsg_listen_all_nsid(sk, msg, skb);

        memset(&scm, 0, sizeof(scm));
        scm.creds = *NETLINK_CREDS(skb);
        if (flags & MSG_TRUNC)
                copied = data_skb->len;

        skb_free_datagram(sk, skb);

        if (READ_ONCE(nlk->cb_running) &&
            atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf / 2) {
                ret = netlink_dump(sk, false);
                if (ret) {
                        WRITE_ONCE(sk->sk_err, -ret);
                        sk_error_report(sk);
                }
        }

        scm_recv(sock, msg, &scm, flags);
out:
        netlink_rcv_wake(sk);
        return err ? : copied;
}

static void netlink_data_ready(struct sock *sk)
{
        BUG();
}

/*
 *        We export these functions to other modules. They provide a
 *        complete set of kernel non-blocking support for message
 *        queueing.
 */

struct sock *
__netlink_kernel_create(struct net *net, int unit, struct module *module,
                        struct netlink_kernel_cfg *cfg)
{
        struct socket *sock;
        struct sock *sk;
        struct netlink_sock *nlk;
        struct listeners *listeners = NULL;
        unsigned int groups;

        BUG_ON(!nl_table);

        if (unit < 0 || unit >= MAX_LINKS)
                return NULL;

        if (sock_create_lite(PF_NETLINK, SOCK_DGRAM, unit, &sock))
                return NULL;

        if (__netlink_create(net, sock, unit, 1) < 0)
                goto out_sock_release_nosk;

        sk = sock->sk;

        if (!cfg || cfg->groups < 32)
                groups = 32;
        else
                groups = cfg->groups;

        listeners = kzalloc(sizeof(*listeners) + NLGRPSZ(groups), GFP_KERNEL);
        if (!listeners)
                goto out_sock_release;

        sk->sk_data_ready = netlink_data_ready;
        if (cfg && cfg->input)
                nlk_sk(sk)->netlink_rcv = cfg->input;

        if (netlink_insert(sk, 0))
                goto out_sock_release;

        nlk = nlk_sk(sk);
        set_bit(NETLINK_F_KERNEL_SOCKET, &nlk->flags);

        netlink_table_grab();
        if (!nl_table[unit].registered) {
                nl_table[unit].groups = groups;
                rcu_assign_pointer(nl_table[unit].listeners, listeners);
                nl_table[unit].module = module;
                if (cfg) {
                        nl_table[unit].bind = cfg->bind;
                        nl_table[unit].unbind = cfg->unbind;
                        nl_table[unit].release = cfg->release;
                        nl_table[unit].flags = cfg->flags;
                }
                nl_table[unit].registered = 1;
        } else {
                kfree(listeners);
                nl_table[unit].registered++;
        }
        netlink_table_ungrab();
        return sk;

out_sock_release:
        kfree(listeners);
        netlink_kernel_release(sk);
        return NULL;

out_sock_release_nosk:
        sock_release(sock);
        return NULL;
}
EXPORT_SYMBOL(__netlink_kernel_create);

void
netlink_kernel_release(struct sock *sk)
{
        if (sk == NULL || sk->sk_socket == NULL)
                return;

        sock_release(sk->sk_socket);
}
EXPORT_SYMBOL(netlink_kernel_release);

int __netlink_change_ngroups(struct sock *sk, unsigned int groups)
{
        struct listeners *new, *old;
        struct netlink_table *tbl = &nl_table[sk->sk_protocol];

        if (groups < 32)
                groups = 32;

        if (NLGRPSZ(tbl->groups) < NLGRPSZ(groups)) {
                new = kzalloc(sizeof(*new) + NLGRPSZ(groups), GFP_ATOMIC);
                if (!new)
                        return -ENOMEM;
                old = nl_deref_protected(tbl->listeners);
                memcpy(new->masks, old->masks, NLGRPSZ(tbl->groups));
                rcu_assign_pointer(tbl->listeners, new);

                kfree_rcu(old, rcu);
        }
        tbl->groups = groups;

        return 0;
}

/**
 * netlink_change_ngroups - change number of multicast groups
 *
 * This changes the number of multicast groups that are available
 * on a certain netlink family. Note that it is not possible to
 * change the number of groups to below 32. Also note that it does
 * not implicitly call netlink_clear_multicast_users() when the
 * number of groups is reduced.
 *
 * @sk: The kernel netlink socket, as returned by netlink_kernel_create().
 * @groups: The new number of groups.
 */
int netlink_change_ngroups(struct sock *sk, unsigned int groups)
{
        int err;

        netlink_table_grab();
        err = __netlink_change_ngroups(sk, groups);
        netlink_table_ungrab();

        return err;
}

void __netlink_clear_multicast_users(struct sock *ksk, unsigned int group)
{
        struct sock *sk;
        struct netlink_table *tbl = &nl_table[ksk->sk_protocol];
        struct hlist_node *tmp;

        sk_for_each_bound_safe(sk, tmp, &tbl->mc_list)
                netlink_update_socket_mc(nlk_sk(sk), group, 0);
}

struct nlmsghdr *
__nlmsg_put(struct sk_buff *skb, u32 portid, u32 seq, int type, int len, int flags)
{
        struct nlmsghdr *nlh;
        int size = nlmsg_msg_size(len);

        nlh = skb_put(skb, NLMSG_ALIGN(size));
        nlh->nlmsg_type = type;
        nlh->nlmsg_len = size;
        nlh->nlmsg_flags = flags;
        nlh->nlmsg_pid = portid;
        nlh->nlmsg_seq = seq;
        if (!__builtin_constant_p(size) || NLMSG_ALIGN(size) - size != 0)
                memset(nlmsg_data(nlh) + len, 0, NLMSG_ALIGN(size) - size);
        return nlh;
}
EXPORT_SYMBOL(__nlmsg_put);

static size_t
netlink_ack_tlv_len(struct netlink_sock *nlk, int err,
                    const struct netlink_ext_ack *extack)
{
        size_t tlvlen;

        if (!extack || !test_bit(NETLINK_F_EXT_ACK, &nlk->flags))
                return 0;

        tlvlen = 0;
        if (extack->_msg)
                tlvlen += nla_total_size(strlen(extack->_msg) + 1);
        if (extack->cookie_len)
                tlvlen += nla_total_size(extack->cookie_len);

        /* Following attributes are only reported as error (not warning) */
        if (!err)
                return tlvlen;

        if (extack->bad_attr)
                tlvlen += nla_total_size(sizeof(u32));
        if (extack->policy)
                tlvlen += netlink_policy_dump_attr_size_estimate(extack->policy);
        if (extack->miss_type)
                tlvlen += nla_total_size(sizeof(u32));
        if (extack->miss_nest)
                tlvlen += nla_total_size(sizeof(u32));

        return tlvlen;
}

static bool nlmsg_check_in_payload(const struct nlmsghdr *nlh, const void *addr)
{
        return !WARN_ON(addr < nlmsg_data(nlh) ||
                        addr - (const void *) nlh >= nlh->nlmsg_len);
}

static void
netlink_ack_tlv_fill(struct sk_buff *skb, const struct nlmsghdr *nlh, int err,
                     const struct netlink_ext_ack *extack)
{
        if (extack->_msg)
                WARN_ON(nla_put_string(skb, NLMSGERR_ATTR_MSG, extack->_msg));
        if (extack->cookie_len)
                WARN_ON(nla_put(skb, NLMSGERR_ATTR_COOKIE,
                                extack->cookie_len, extack->cookie));

        if (!err)
                return;

        if (extack->bad_attr && nlmsg_check_in_payload(nlh, extack->bad_attr))
                WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_OFFS,
                                    (u8 *)extack->bad_attr - (const u8 *)nlh));
        if (extack->policy)
                netlink_policy_dump_write_attr(skb, extack->policy,
                                               NLMSGERR_ATTR_POLICY);
        if (extack->miss_type)
                WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_MISS_TYPE,
                                    extack->miss_type));
        if (extack->miss_nest && nlmsg_check_in_payload(nlh, extack->miss_nest))
                WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_MISS_NEST,
                                    (u8 *)extack->miss_nest - (const u8 *)nlh));
}

/*
 * It looks a bit ugly.
 * It would be better to create kernel thread.
 */

static int netlink_dump_done(struct netlink_sock *nlk, struct sk_buff *skb,
                             struct netlink_callback *cb,
                             struct netlink_ext_ack *extack)
{
        struct nlmsghdr *nlh;
        size_t extack_len;

        nlh = nlmsg_put_answer(skb, cb, NLMSG_DONE, sizeof(nlk->dump_done_errno),
                               NLM_F_MULTI | cb->answer_flags);
        if (WARN_ON(!nlh))
                return -ENOBUFS;

        nl_dump_check_consistent(cb, nlh);
        memcpy(nlmsg_data(nlh), &nlk->dump_done_errno, sizeof(nlk->dump_done_errno));

        extack_len = netlink_ack_tlv_len(nlk, nlk->dump_done_errno, extack);
        if (extack_len) {
                nlh->nlmsg_flags |= NLM_F_ACK_TLVS;
                if (skb_tailroom(skb) >= extack_len) {
                        netlink_ack_tlv_fill(skb, cb->nlh,
                                             nlk->dump_done_errno, extack);
                        nlmsg_end(skb, nlh);
                }
        }

        return 0;
}

static int netlink_dump(struct sock *sk, bool lock_taken)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        struct netlink_ext_ack extack = {};
        struct netlink_callback *cb;
        struct sk_buff *skb = NULL;
        unsigned int rmem, rcvbuf;
        size_t max_recvmsg_len;
        struct module *module;
        int err = -ENOBUFS;
        int alloc_min_size;
        int alloc_size;

        if (!lock_taken)
                mutex_lock(&nlk->nl_cb_mutex);
        if (!nlk->cb_running) {
                err = -EINVAL;
                goto errout_skb;
        }

        /* NLMSG_GOODSIZE is small to avoid high order allocations being
         * required, but it makes sense to _attempt_ a 32KiB allocation
         * to reduce number of system calls on dump operations, if user
         * ever provided a big enough buffer.
         */
        cb = &nlk->cb;
        alloc_min_size = max_t(int, cb->min_dump_alloc, NLMSG_GOODSIZE);

        max_recvmsg_len = READ_ONCE(nlk->max_recvmsg_len);
        if (alloc_min_size < max_recvmsg_len) {
                alloc_size = max_recvmsg_len;
                skb = alloc_skb(alloc_size,
                                (GFP_KERNEL & ~__GFP_DIRECT_RECLAIM) |
                                __GFP_NOWARN | __GFP_NORETRY);
        }
        if (!skb) {
                alloc_size = alloc_min_size;
                skb = alloc_skb(alloc_size, GFP_KERNEL);
        }
        if (!skb)
                goto errout_skb;

        rcvbuf = READ_ONCE(sk->sk_rcvbuf);
        rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc);
        if (rmem != skb->truesize && rmem >= rcvbuf) {
                atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
                goto errout_skb;
        }

        /* Trim skb to allocated size. User is expected to provide buffer as
         * large as max(min_dump_alloc, 32KiB (max_recvmsg_len capped at
         * netlink_recvmsg())). dump will pack as many smaller messages as
         * could fit within the allocated skb. skb is typically allocated
         * with larger space than required (could be as much as near 2x the
         * requested size with align to next power of 2 approach). Allowing
         * dump to use the excess space makes it difficult for a user to have a
         * reasonable static buffer based on the expected largest dump of a
         * single netdev. The outcome is MSG_TRUNC error.
         */
        skb_reserve(skb, skb_tailroom(skb) - alloc_size);

        /* Make sure malicious BPF programs can not read unitialized memory
         * from skb->head -> skb->data
         */
        skb_reset_network_header(skb);
        skb_reset_mac_header(skb);

        netlink_skb_set_owner_r(skb, sk);

        if (nlk->dump_done_errno > 0) {
                cb->extack = &extack;

                nlk->dump_done_errno = cb->dump(skb, cb);

                /* EMSGSIZE plus something already in the skb means
                 * that there's more to dump but current skb has filled up.
                 * If the callback really wants to return EMSGSIZE to user space
                 * it needs to do so again, on the next cb->dump() call,
                 * without putting data in the skb.
                 */
                if (nlk->dump_done_errno == -EMSGSIZE && skb->len)
                        nlk->dump_done_errno = skb->len;

                cb->extack = NULL;
        }

        if (nlk->dump_done_errno > 0 ||
            skb_tailroom(skb) < nlmsg_total_size(sizeof(nlk->dump_done_errno))) {
                mutex_unlock(&nlk->nl_cb_mutex);

                if (sk_filter(sk, skb))
                        kfree_skb(skb);
                else
                        __netlink_sendskb(sk, skb);
                return 0;
        }

        if (netlink_dump_done(nlk, skb, cb, &extack))
                goto errout_skb;

#ifdef CONFIG_COMPAT_NETLINK_MESSAGES
        /* frag_list skb's data is used for compat tasks
         * and the regular skb's data for normal (non-compat) tasks.
         * See netlink_recvmsg().
         */
        if (unlikely(skb_shinfo(skb)->frag_list)) {
                if (netlink_dump_done(nlk, skb_shinfo(skb)->frag_list, cb, &extack))
                        goto errout_skb;
        }
#endif

        if (sk_filter(sk, skb))
                kfree_skb(skb);
        else
                __netlink_sendskb(sk, skb);

        if (cb->done)
                cb->done(cb);

        WRITE_ONCE(nlk->cb_running, false);
        module = cb->module;
        skb = cb->skb;
        mutex_unlock(&nlk->nl_cb_mutex);
        module_put(module);
        consume_skb(skb);
        return 0;

errout_skb:
        mutex_unlock(&nlk->nl_cb_mutex);
        kfree_skb(skb);
        return err;
}

int __netlink_dump_start(struct sock *ssk, struct sk_buff *skb,
                         const struct nlmsghdr *nlh,
                         struct netlink_dump_control *control)
{
        struct netlink_callback *cb;
        struct netlink_sock *nlk;
        struct sock *sk;
        int ret;

        refcount_inc(&skb->users);

        sk = netlink_lookup(sock_net(ssk), ssk->sk_protocol, NETLINK_CB(skb).portid);
        if (sk == NULL) {
                ret = -ECONNREFUSED;
                goto error_free;
        }

        nlk = nlk_sk(sk);
        mutex_lock(&nlk->nl_cb_mutex);
        /* A dump is in progress... */
        if (nlk->cb_running) {
                ret = -EBUSY;
                goto error_unlock;
        }
        /* add reference of module which cb->dump belongs to */
        if (!try_module_get(control->module)) {
                ret = -EPROTONOSUPPORT;
                goto error_unlock;
        }

        cb = &nlk->cb;
        memset(cb, 0, sizeof(*cb));
        cb->dump = control->dump;
        cb->done = control->done;
        cb->nlh = nlh;
        cb->data = control->data;
        cb->module = control->module;
        cb->min_dump_alloc = control->min_dump_alloc;
        cb->flags = control->flags;
        cb->skb = skb;

        cb->strict_check = nlk_test_bit(STRICT_CHK, NETLINK_CB(skb).sk);

        if (control->start) {
                cb->extack = control->extack;
                ret = control->start(cb);
                cb->extack = NULL;
                if (ret)
                        goto error_put;
        }

        WRITE_ONCE(nlk->cb_running, true);
        nlk->dump_done_errno = INT_MAX;

        ret = netlink_dump(sk, true);

        sock_put(sk);

        if (ret)
                return ret;

        /* We successfully started a dump, by returning -EINTR we
         * signal not to send ACK even if it was requested.
         */
        return -EINTR;

error_put:
        module_put(control->module);
error_unlock:
        sock_put(sk);
        mutex_unlock(&nlk->nl_cb_mutex);
error_free:
        kfree_skb(skb);
        return ret;
}
EXPORT_SYMBOL(__netlink_dump_start);

void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err,
                 const struct netlink_ext_ack *extack)
{
        struct sk_buff *skb;
        struct nlmsghdr *rep;
        struct nlmsgerr *errmsg;
        size_t payload = sizeof(*errmsg);
        struct netlink_sock *nlk = nlk_sk(NETLINK_CB(in_skb).sk);
        unsigned int flags = 0;
        size_t tlvlen;

        /* Error messages get the original request appended, unless the user
         * requests to cap the error message, and get extra error data if
         * requested.
         */
        if (err && !test_bit(NETLINK_F_CAP_ACK, &nlk->flags))
                payload += nlmsg_len(nlh);
        else
                flags |= NLM_F_CAPPED;

        tlvlen = netlink_ack_tlv_len(nlk, err, extack);
        if (tlvlen)
                flags |= NLM_F_ACK_TLVS;

        skb = nlmsg_new(payload + tlvlen, GFP_KERNEL);
        if (!skb)
                goto err_skb;

        rep = nlmsg_put(skb, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
                        NLMSG_ERROR, sizeof(*errmsg), flags);
        if (!rep)
                goto err_bad_put;
        errmsg = nlmsg_data(rep);
        errmsg->error = err;
        errmsg->msg = *nlh;

        if (!(flags & NLM_F_CAPPED)) {
                if (!nlmsg_append(skb, nlmsg_len(nlh)))
                        goto err_bad_put;

                memcpy(nlmsg_data(&errmsg->msg), nlmsg_data(nlh),
                       nlmsg_len(nlh));
        }

        if (tlvlen)
                netlink_ack_tlv_fill(skb, nlh, err, extack);

        nlmsg_end(skb, rep);

        nlmsg_unicast(in_skb->sk, skb, NETLINK_CB(in_skb).portid);

        return;

err_bad_put:
        nlmsg_free(skb);
err_skb:
        WRITE_ONCE(NETLINK_CB(in_skb).sk->sk_err, ENOBUFS);
        sk_error_report(NETLINK_CB(in_skb).sk);
}
EXPORT_SYMBOL(netlink_ack);

int netlink_rcv_skb(struct sk_buff *skb, int (*cb)(struct sk_buff *,
                                                   struct nlmsghdr *,
                                                   struct netlink_ext_ack *))
{
        struct netlink_ext_ack extack;
        struct nlmsghdr *nlh;
        int err;

        while (skb->len >= nlmsg_total_size(0)) {
                int msglen;

                memset(&extack, 0, sizeof(extack));
                nlh = nlmsg_hdr(skb);
                err = 0;

                if (nlh->nlmsg_len < NLMSG_HDRLEN || skb->len < nlh->nlmsg_len)
                        return 0;

                /* Only requests are handled by the kernel */
                if (!(nlh->nlmsg_flags & NLM_F_REQUEST))
                        goto ack;

                /* Skip control messages */
                if (nlh->nlmsg_type < NLMSG_MIN_TYPE)
                        goto ack;

                err = cb(skb, nlh, &extack);
                if (err == -EINTR)
                        goto skip;

ack:
                if (nlh->nlmsg_flags & NLM_F_ACK || err)
                        netlink_ack(skb, nlh, err, &extack);

skip:
                msglen = NLMSG_ALIGN(nlh->nlmsg_len);
                if (msglen > skb->len)
                        msglen = skb->len;
                skb_pull(skb, msglen);
        }

        return 0;
}
EXPORT_SYMBOL(netlink_rcv_skb);

/**
 * nlmsg_notify - send a notification netlink message
 * @sk: netlink socket to use
 * @skb: notification message
 * @portid: destination netlink portid for reports or 0
 * @group: destination multicast group or 0
 * @report: 1 to report back, 0 to disable
 * @flags: allocation flags
 */
int nlmsg_notify(struct sock *sk, struct sk_buff *skb, u32 portid,
                 unsigned int group, int report, gfp_t flags)
{
        int err = 0;

        if (group) {
                int exclude_portid = 0;

                if (report) {
                        refcount_inc(&skb->users);
                        exclude_portid = portid;
                }

                /* errors reported via destination sk->sk_err, but propagate
                 * delivery errors if NETLINK_BROADCAST_ERROR flag is set */
                err = nlmsg_multicast(sk, skb, exclude_portid, group, flags);
                if (err == -ESRCH)
                        err = 0;
        }

        if (report) {
                int err2;

                err2 = nlmsg_unicast(sk, skb, portid);
                if (!err)
                        err = err2;
        }

        return err;
}
EXPORT_SYMBOL(nlmsg_notify);

#ifdef CONFIG_PROC_FS
struct nl_seq_iter {
        struct seq_net_private p;
        struct rhashtable_iter hti;
        int link;
};

static void netlink_walk_start(struct nl_seq_iter *iter)
{
        rhashtable_walk_enter(&nl_table[iter->link].hash, &iter->hti);
        rhashtable_walk_start(&iter->hti);
}

static void netlink_walk_stop(struct nl_seq_iter *iter)
{
        rhashtable_walk_stop(&iter->hti);
        rhashtable_walk_exit(&iter->hti);
}

static void *__netlink_seq_next(struct seq_file *seq)
{
        struct nl_seq_iter *iter = seq->private;
        struct netlink_sock *nlk;

        do {
                for (;;) {
                        nlk = rhashtable_walk_next(&iter->hti);

                        if (IS_ERR(nlk)) {
                                if (PTR_ERR(nlk) == -EAGAIN)
                                        continue;

                                return nlk;
                        }

                        if (nlk)
                                break;

                        netlink_walk_stop(iter);
                        if (++iter->link >= MAX_LINKS)
                                return NULL;

                        netlink_walk_start(iter);
                }
        } while (sock_net(&nlk->sk) != seq_file_net(seq));

        return nlk;
}

static void *netlink_seq_start(struct seq_file *seq, loff_t *posp)
        __acquires(RCU)
{
        struct nl_seq_iter *iter = seq->private;
        void *obj = SEQ_START_TOKEN;
        loff_t pos;

        iter->link = 0;

        netlink_walk_start(iter);

        for (pos = *posp; pos && obj && !IS_ERR(obj); pos--)
                obj = __netlink_seq_next(seq);

        return obj;
}

static void *netlink_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        ++*pos;
        return __netlink_seq_next(seq);
}

static void netlink_native_seq_stop(struct seq_file *seq, void *v)
{
        struct nl_seq_iter *iter = seq->private;

        if (iter->link >= MAX_LINKS)
                return;

        netlink_walk_stop(iter);
}


static int netlink_native_seq_show(struct seq_file *seq, void *v)
{
        if (v == SEQ_START_TOKEN) {
                seq_puts(seq,
                         "sk               Eth Pid        Groups   "
                         "Rmem     Wmem     Dump  Locks    Drops    Inode\n");
        } else {
                struct sock *s = v;
                struct netlink_sock *nlk = nlk_sk(s);

                seq_printf(seq, "%pK %-3d %-10u %08x %-8d %-8d %-5d %-8d %-8u %-8lu\n",
                           s,
                           s->sk_protocol,
                           nlk->portid,
                           nlk->groups ? (u32)nlk->groups[0] : 0,
                           sk_rmem_alloc_get(s),
                           sk_wmem_alloc_get(s),
                           READ_ONCE(nlk->cb_running),
                           refcount_read(&s->sk_refcnt),
                           sk_drops_read(s),
                           sock_i_ino(s)
                        );

        }
        return 0;
}

#ifdef CONFIG_BPF_SYSCALL
struct bpf_iter__netlink {
        __bpf_md_ptr(struct bpf_iter_meta *, meta);
        __bpf_md_ptr(struct netlink_sock *, sk);
};

DEFINE_BPF_ITER_FUNC(netlink, struct bpf_iter_meta *meta, struct netlink_sock *sk)

static int netlink_prog_seq_show(struct bpf_prog *prog,
                                  struct bpf_iter_meta *meta,
                                  void *v)
{
        struct bpf_iter__netlink ctx;

        meta->seq_num--;  /* skip SEQ_START_TOKEN */
        ctx.meta = meta;
        ctx.sk = nlk_sk((struct sock *)v);
        return bpf_iter_run_prog(prog, &ctx);
}

static int netlink_seq_show(struct seq_file *seq, void *v)
{
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;

        meta.seq = seq;
        prog = bpf_iter_get_info(&meta, false);
        if (!prog)
                return netlink_native_seq_show(seq, v);

        if (v != SEQ_START_TOKEN)
                return netlink_prog_seq_show(prog, &meta, v);

        return 0;
}

static void netlink_seq_stop(struct seq_file *seq, void *v)
{
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;

        if (!v) {
                meta.seq = seq;
                prog = bpf_iter_get_info(&meta, true);
                if (prog)
                        (void)netlink_prog_seq_show(prog, &meta, v);
        }

        netlink_native_seq_stop(seq, v);
}
#else
static int netlink_seq_show(struct seq_file *seq, void *v)
{
        return netlink_native_seq_show(seq, v);
}

static void netlink_seq_stop(struct seq_file *seq, void *v)
{
        netlink_native_seq_stop(seq, v);
}
#endif

static const struct seq_operations netlink_seq_ops = {
        .start  = netlink_seq_start,
        .next   = netlink_seq_next,
        .stop   = netlink_seq_stop,
        .show   = netlink_seq_show,
};
#endif

int netlink_register_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&netlink_chain, nb);
}
EXPORT_SYMBOL(netlink_register_notifier);

int netlink_unregister_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&netlink_chain, nb);
}
EXPORT_SYMBOL(netlink_unregister_notifier);

static const struct proto_ops netlink_ops = {
        .family =        PF_NETLINK,
        .owner =        THIS_MODULE,
        .release =        netlink_release,
        .bind =                netlink_bind,
        .connect =        netlink_connect,
        .socketpair =        sock_no_socketpair,
        .accept =        sock_no_accept,
        .getname =        netlink_getname,
        .poll =                datagram_poll,
        .ioctl =        netlink_ioctl,
        .listen =        sock_no_listen,
        .shutdown =        sock_no_shutdown,
        .setsockopt =        netlink_setsockopt,
        .getsockopt =        netlink_getsockopt,
        .sendmsg =        netlink_sendmsg,
        .recvmsg =        netlink_recvmsg,
        .mmap =                sock_no_mmap,
};

static const struct net_proto_family netlink_family_ops = {
        .family = PF_NETLINK,
        .create = netlink_create,
        .owner        = THIS_MODULE,        /* for consistency 8) */
};

static int __net_init netlink_net_init(struct net *net)
{
#ifdef CONFIG_PROC_FS
        if (!proc_create_net("netlink", 0, net->proc_net, &netlink_seq_ops,
                        sizeof(struct nl_seq_iter)))
                return -ENOMEM;
#endif
        return 0;
}

static void __net_exit netlink_net_exit(struct net *net)
{
#ifdef CONFIG_PROC_FS
        remove_proc_entry("netlink", net->proc_net);
#endif
}

static void __init netlink_add_usersock_entry(void)
{
        struct listeners *listeners;
        int groups = 32;

        listeners = kzalloc(sizeof(*listeners) + NLGRPSZ(groups), GFP_KERNEL);
        if (!listeners)
                panic("netlink_add_usersock_entry: Cannot allocate listeners\n");

        netlink_table_grab();

        nl_table[NETLINK_USERSOCK].groups = groups;
        rcu_assign_pointer(nl_table[NETLINK_USERSOCK].listeners, listeners);
        nl_table[NETLINK_USERSOCK].module = THIS_MODULE;
        nl_table[NETLINK_USERSOCK].registered = 1;
        nl_table[NETLINK_USERSOCK].flags = NL_CFG_F_NONROOT_SEND;

        netlink_table_ungrab();
}

static struct pernet_operations __net_initdata netlink_net_ops = {
        .init = netlink_net_init,
        .exit = netlink_net_exit,
};

static inline u32 netlink_hash(const void *data, u32 len, u32 seed)
{
        const struct netlink_sock *nlk = data;
        struct netlink_compare_arg arg;

        netlink_compare_arg_init(&arg, sock_net(&nlk->sk), nlk->portid);
        return jhash2((u32 *)&arg, netlink_compare_arg_len / sizeof(u32), seed);
}

static const struct rhashtable_params netlink_rhashtable_params = {
        .head_offset = offsetof(struct netlink_sock, node),
        .key_len = netlink_compare_arg_len,
        .obj_hashfn = netlink_hash,
        .obj_cmpfn = netlink_compare,
        .automatic_shrinking = true,
};

#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
BTF_ID_LIST_SINGLE(btf_netlink_sock_id, struct, netlink_sock)

static const struct bpf_iter_seq_info netlink_seq_info = {
        .seq_ops                = &netlink_seq_ops,
        .init_seq_private        = bpf_iter_init_seq_net,
        .fini_seq_private        = bpf_iter_fini_seq_net,
        .seq_priv_size                = sizeof(struct nl_seq_iter),
};

static struct bpf_iter_reg netlink_reg_info = {
        .target                        = "netlink",
        .ctx_arg_info_size        = 1,
        .ctx_arg_info                = {
                { offsetof(struct bpf_iter__netlink, sk),
                  PTR_TO_BTF_ID_OR_NULL },
        },
        .seq_info                = &netlink_seq_info,
};

static int __init bpf_iter_register(void)
{
        netlink_reg_info.ctx_arg_info[0].btf_id = *btf_netlink_sock_id;
        return bpf_iter_reg_target(&netlink_reg_info);
}
#endif

static int __init netlink_proto_init(void)
{
        int i;
        int err = proto_register(&netlink_proto, 0);

        if (err != 0)
                goto out;

#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
        err = bpf_iter_register();
        if (err)
                goto out;
#endif

        BUILD_BUG_ON(sizeof(struct netlink_skb_parms) > sizeof_field(struct sk_buff, cb));

        nl_table = kcalloc(MAX_LINKS, sizeof(*nl_table), GFP_KERNEL);
        if (!nl_table)
                goto panic;

        for (i = 0; i < MAX_LINKS; i++) {
                if (rhashtable_init(&nl_table[i].hash,
                                    &netlink_rhashtable_params) < 0)
                        goto panic;
        }

        netlink_add_usersock_entry();

        sock_register(&netlink_family_ops);
        register_pernet_subsys(&netlink_net_ops);
        register_pernet_subsys(&netlink_tap_net_ops);
        /* The netlink device handler may be needed early. */
        rtnetlink_init();
out:
        return err;
panic:
        panic("netlink_init: Cannot allocate nl_table\n");
}

core_initcall(netlink_proto_init);















































   12 









    3 



    1 


    2 
    1 














   10 









    6 

   10 





    4 



    7 







   11 



   11 






    7 

    6 
    1 






































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
// SPDX-License-Identifier: GPL-2.0-only
/*
 * GHASH: hash function for GCM (Galois/Counter Mode).
 *
 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
 * Copyright (c) 2009 Intel Corp.
 *   Author: Huang Ying <ying.huang@intel.com>
 */

/*
 * GHASH is a keyed hash function used in GCM authentication tag generation.
 *
 * The original GCM paper [1] presents GHASH as a function GHASH(H, A, C) which
 * takes a 16-byte hash key H, additional authenticated data A, and a ciphertext
 * C.  It formats A and C into a single byte string X, interprets X as a
 * polynomial over GF(2^128), and evaluates this polynomial at the point H.
 *
 * However, the NIST standard for GCM [2] presents GHASH as GHASH(H, X) where X
 * is the already-formatted byte string containing both A and C.
 *
 * "ghash" in the Linux crypto API uses the 'X' (pre-formatted) convention,
 * since the API supports only a single data stream per hash.  Thus, the
 * formatting of 'A' and 'C' is done in the "gcm" template, not in "ghash".
 *
 * The reason "ghash" is separate from "gcm" is to allow "gcm" to use an
 * accelerated "ghash" when a standalone accelerated "gcm(aes)" is unavailable.
 * It is generally inappropriate to use "ghash" for other purposes, since it is
 * an "ε-almost-XOR-universal hash function", not a cryptographic hash function.
 * It can only be used securely in crypto modes specially designed to use it.
 *
 * [1] The Galois/Counter Mode of Operation (GCM)
 *     (http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.694.695&rep=rep1&type=pdf)
 * [2] Recommendation for Block Cipher Modes of Operation: Galois/Counter Mode (GCM) and GMAC
 *     (https://csrc.nist.gov/publications/detail/sp/800-38d/final)
 */

#include <crypto/gf128mul.h>
#include <crypto/ghash.h>
#include <crypto/internal/hash.h>
#include <crypto/utils.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>

static int ghash_init(struct shash_desc *desc)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);

        memset(dctx, 0, sizeof(*dctx));

        return 0;
}

static int ghash_setkey(struct crypto_shash *tfm,
                        const u8 *key, unsigned int keylen)
{
        struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
        be128 k;

        if (keylen != GHASH_BLOCK_SIZE)
                return -EINVAL;

        if (ctx->gf128)
                gf128mul_free_4k(ctx->gf128);

        BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE);
        memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */
        ctx->gf128 = gf128mul_init_4k_lle(&k);
        memzero_explicit(&k, GHASH_BLOCK_SIZE);

        if (!ctx->gf128)
                return -ENOMEM;

        return 0;
}

static int ghash_update(struct shash_desc *desc,
                         const u8 *src, unsigned int srclen)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
        struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
        u8 *dst = dctx->buffer;

        do {
                crypto_xor(dst, src, GHASH_BLOCK_SIZE);
                gf128mul_4k_lle((be128 *)dst, ctx->gf128);
                src += GHASH_BLOCK_SIZE;
                srclen -= GHASH_BLOCK_SIZE;
        } while (srclen >= GHASH_BLOCK_SIZE);

        return srclen;
}

static void ghash_flush(struct shash_desc *desc, const u8 *src,
                        unsigned int len)
{
        struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
        u8 *dst = dctx->buffer;

        if (len) {
                crypto_xor(dst, src, len);
                gf128mul_4k_lle((be128 *)dst, ctx->gf128);
        }
}

static int ghash_finup(struct shash_desc *desc, const u8 *src,
                       unsigned int len, u8 *dst)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
        u8 *buf = dctx->buffer;

        ghash_flush(desc, src, len);
        memcpy(dst, buf, GHASH_BLOCK_SIZE);

        return 0;
}

static void ghash_exit_tfm(struct crypto_tfm *tfm)
{
        struct ghash_ctx *ctx = crypto_tfm_ctx(tfm);
        if (ctx->gf128)
                gf128mul_free_4k(ctx->gf128);
}

static struct shash_alg ghash_alg = {
        .digestsize        = GHASH_DIGEST_SIZE,
        .init                = ghash_init,
        .update                = ghash_update,
        .finup                = ghash_finup,
        .setkey                = ghash_setkey,
        .descsize        = sizeof(struct ghash_desc_ctx),
        .base                = {
                .cra_name                = "ghash",
                .cra_driver_name        = "ghash-generic",
                .cra_priority                = 100,
                .cra_flags                = CRYPTO_AHASH_ALG_BLOCK_ONLY,
                .cra_blocksize                = GHASH_BLOCK_SIZE,
                .cra_ctxsize                = sizeof(struct ghash_ctx),
                .cra_module                = THIS_MODULE,
                .cra_exit                = ghash_exit_tfm,
        },
};

static int __init ghash_mod_init(void)
{
        return crypto_register_shash(&ghash_alg);
}

static void __exit ghash_mod_exit(void)
{
        crypto_unregister_shash(&ghash_alg);
}

module_init(ghash_mod_init);
module_exit(ghash_mod_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("GHASH hash function");
MODULE_ALIAS_CRYPTO("ghash");
MODULE_ALIAS_CRYPTO("ghash-generic");

























































































































































    3 






    2 
    1 


    1 



























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
// SPDX-License-Identifier: GPL-2.0-only
/* Kernel module to match ROUTING parameters. */

/* (C) 2001-2002 Andras Kis-Szabo <kisza@sch.bme.hu>
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/ipv6.h>
#include <linux/types.h>
#include <net/checksum.h>
#include <net/ipv6.h>

#include <asm/byteorder.h>

#include <linux/netfilter/x_tables.h>
#include <linux/netfilter_ipv6/ip6_tables.h>
#include <linux/netfilter_ipv6/ip6t_rt.h>

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Xtables: IPv6 Routing Header match");
MODULE_AUTHOR("Andras Kis-Szabo <kisza@sch.bme.hu>");

/* Returns 1 if the id is matched by the range, 0 otherwise */
static inline bool
segsleft_match(u_int32_t min, u_int32_t max, u_int32_t id, bool invert)
{
        return (id >= min && id <= max) ^ invert;
}

static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par)
{
        struct ipv6_rt_hdr _route;
        const struct ipv6_rt_hdr *rh;
        const struct ip6t_rt *rtinfo = par->matchinfo;
        unsigned int temp;
        unsigned int ptr = 0;
        unsigned int hdrlen = 0;
        bool ret = false;
        struct in6_addr _addr;
        const struct in6_addr *ap;
        int err;

        err = ipv6_find_hdr(skb, &ptr, NEXTHDR_ROUTING, NULL, NULL);
        if (err < 0) {
                if (err != -ENOENT)
                        par->hotdrop = true;
                return false;
        }

        rh = skb_header_pointer(skb, ptr, sizeof(_route), &_route);
        if (rh == NULL) {
                par->hotdrop = true;
                return false;
        }

        hdrlen = ipv6_optlen(rh);
        if (skb->len - ptr < hdrlen) {
                /* Pcket smaller than its length field */
                return false;
        }

        ret = (segsleft_match(rtinfo->segsleft[0], rtinfo->segsleft[1],
                              rh->segments_left,
                              !!(rtinfo->invflags & IP6T_RT_INV_SGS))) &&
              (!(rtinfo->flags & IP6T_RT_LEN) ||
               ((rtinfo->hdrlen == hdrlen) ^
                !!(rtinfo->invflags & IP6T_RT_INV_LEN))) &&
              (!(rtinfo->flags & IP6T_RT_TYP) ||
               ((rtinfo->rt_type == rh->type) ^
                !!(rtinfo->invflags & IP6T_RT_INV_TYP)));

        if (ret && (rtinfo->flags & IP6T_RT_RES)) {
                const u_int32_t *rp;
                u_int32_t _reserved;
                rp = skb_header_pointer(skb,
                                        ptr + offsetof(struct rt0_hdr,
                                                       reserved),
                                        sizeof(_reserved),
                                        &_reserved);
                if (!rp) {
                        par->hotdrop = true;
                        return false;
                }

                ret = (*rp == 0);
        }

        if (!(rtinfo->flags & IP6T_RT_FST)) {
                return ret;
        } else if (rtinfo->flags & IP6T_RT_FST_NSTRICT) {
                if (rtinfo->addrnr > (unsigned int)((hdrlen - 8) / 16)) {
                        return false;
                } else {
                        unsigned int i = 0;

                        for (temp = 0;
                             temp < (unsigned int)((hdrlen - 8) / 16);
                             temp++) {
                                ap = skb_header_pointer(skb,
                                                        ptr
                                                        + sizeof(struct rt0_hdr)
                                                        + temp * sizeof(_addr),
                                                        sizeof(_addr),
                                                        &_addr);

                                if (ap == NULL) {
                                        par->hotdrop = true;
                                        return false;
                                }

                                if (ipv6_addr_equal(ap, &rtinfo->addrs[i]))
                                        i++;
                                if (i == rtinfo->addrnr)
                                        break;
                        }
                        if (i == rtinfo->addrnr)
                                return ret;
                        else
                                return false;
                }
        } else {
                if (rtinfo->addrnr > (unsigned int)((hdrlen - 8) / 16)) {
                        return false;
                } else {
                        for (temp = 0; temp < rtinfo->addrnr; temp++) {
                                ap = skb_header_pointer(skb,
                                                        ptr
                                                        + sizeof(struct rt0_hdr)
                                                        + temp * sizeof(_addr),
                                                        sizeof(_addr),
                                                        &_addr);
                                if (ap == NULL) {
                                        par->hotdrop = true;
                                        return false;
                                }

                                if (!ipv6_addr_equal(ap, &rtinfo->addrs[temp]))
                                        break;
                        }
                        if (temp == rtinfo->addrnr &&
                            temp == (unsigned int)((hdrlen - 8) / 16))
                                return ret;
                        else
                                return false;
                }
        }

        return false;
}

static int rt_mt6_check(const struct xt_mtchk_param *par)
{
        const struct ip6t_rt *rtinfo = par->matchinfo;

        if (rtinfo->invflags & ~IP6T_RT_INV_MASK) {
                pr_debug("unknown flags %X\n", rtinfo->invflags);
                return -EINVAL;
        }
        if ((rtinfo->flags & (IP6T_RT_RES | IP6T_RT_FST_MASK)) &&
            (!(rtinfo->flags & IP6T_RT_TYP) ||
             (rtinfo->rt_type != 0) ||
             (rtinfo->invflags & IP6T_RT_INV_TYP))) {
                pr_debug("`--rt-type 0' required before `--rt-0-*'");
                return -EINVAL;
        }

        return 0;
}

static struct xt_match rt_mt6_reg __read_mostly = {
        .name                = "rt",
        .family                = NFPROTO_IPV6,
        .match                = rt_mt6,
        .matchsize        = sizeof(struct ip6t_rt),
        .checkentry        = rt_mt6_check,
        .me                = THIS_MODULE,
};

static int __init rt_mt6_init(void)
{
        return xt_register_match(&rt_mt6_reg);
}

static void __exit rt_mt6_exit(void)
{
        xt_unregister_match(&rt_mt6_reg);
}

module_init(rt_mt6_init);
module_exit(rt_mt6_exit);







































































































































































































































































































































    9 




    9 

    9 
















    9 






















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Module kallsyms support
 *
 * Copyright (C) 2010 Rusty Russell
 */

#include <linux/module.h>
#include <linux/module_symbol.h>
#include <linux/kallsyms.h>
#include <linux/buildid.h>
#include <linux/bsearch.h>
#include "internal.h"

/* Lookup exported symbol in given range of kernel_symbols */
static const struct kernel_symbol *lookup_exported_symbol(const char *name,
                                                          const struct kernel_symbol *start,
                                                          const struct kernel_symbol *stop)
{
        return bsearch(name, start, stop - start,
                        sizeof(struct kernel_symbol), cmp_name);
}

static int is_exported(const char *name, unsigned long value,
                       const struct module *mod)
{
        const struct kernel_symbol *ks;

        if (!mod)
                ks = lookup_exported_symbol(name, __start___ksymtab, __stop___ksymtab);
        else
                ks = lookup_exported_symbol(name, mod->syms, mod->syms + mod->num_syms);

        return ks && kernel_symbol_value(ks) == value;
}

/* As per nm */
static char elf_type(const Elf_Sym *sym, const struct load_info *info)
{
        const Elf_Shdr *sechdrs = info->sechdrs;

        if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
                if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT)
                        return 'v';
                else
                        return 'w';
        }
        if (sym->st_shndx == SHN_UNDEF)
                return 'U';
        if (sym->st_shndx == SHN_ABS || sym->st_shndx == info->index.pcpu)
                return 'a';
        if (sym->st_shndx >= SHN_LORESERVE)
                return '?';
        if (sechdrs[sym->st_shndx].sh_flags & SHF_EXECINSTR)
                return 't';
        if (sechdrs[sym->st_shndx].sh_flags & SHF_ALLOC &&
            sechdrs[sym->st_shndx].sh_type != SHT_NOBITS) {
                if (!(sechdrs[sym->st_shndx].sh_flags & SHF_WRITE))
                        return 'r';
                else if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
                        return 'g';
                else
                        return 'd';
        }
        if (sechdrs[sym->st_shndx].sh_type == SHT_NOBITS) {
                if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
                        return 's';
                else
                        return 'b';
        }
        if (strstarts(info->secstrings + sechdrs[sym->st_shndx].sh_name,
                      ".debug")) {
                return 'n';
        }
        return '?';
}

static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
                           unsigned int shnum, unsigned int pcpundx)
{
        const Elf_Shdr *sec;
        enum mod_mem_type type;

        if (src->st_shndx == SHN_UNDEF ||
            src->st_shndx >= shnum ||
            !src->st_name)
                return false;

#ifdef CONFIG_KALLSYMS_ALL
        if (src->st_shndx == pcpundx)
                return true;
#endif

        sec = sechdrs + src->st_shndx;
        type = sec->sh_entsize >> SH_ENTSIZE_TYPE_SHIFT;
        if (!(sec->sh_flags & SHF_ALLOC)
#ifndef CONFIG_KALLSYMS_ALL
            || !(sec->sh_flags & SHF_EXECINSTR)
#endif
            || mod_mem_type_is_init(type))
                return false;

        return true;
}

/*
 * We only allocate and copy the strings needed by the parts of symtab
 * we keep.  This is simple, but has the effect of making multiple
 * copies of duplicates.  We could be more sophisticated, see
 * linux-kernel thread starting with
 * <73defb5e4bca04a6431392cc341112b1@localhost>.
 */
void layout_symtab(struct module *mod, struct load_info *info)
{
        Elf_Shdr *symsect = info->sechdrs + info->index.sym;
        Elf_Shdr *strsect = info->sechdrs + info->index.str;
        const Elf_Sym *src;
        unsigned int i, nsrc, ndst, strtab_size = 0;
        struct module_memory *mod_mem_data = &mod->mem[MOD_DATA];
        struct module_memory *mod_mem_init_data = &mod->mem[MOD_INIT_DATA];

        /* Put symbol section at end of init part of module. */
        symsect->sh_flags |= SHF_ALLOC;
        symsect->sh_entsize = module_get_offset_and_type(mod, MOD_INIT_DATA,
                                                         symsect, info->index.sym);
        pr_debug("\t%s\n", info->secstrings + symsect->sh_name);

        src = (void *)info->hdr + symsect->sh_offset;
        nsrc = symsect->sh_size / sizeof(*src);

        /* Compute total space required for the core symbols' strtab. */
        for (ndst = i = 0; i < nsrc; i++) {
                if (i == 0 || is_livepatch_module(mod) ||
                    is_core_symbol(src + i, info->sechdrs, info->hdr->e_shnum,
                                   info->index.pcpu)) {
                        strtab_size += strlen(&info->strtab[src[i].st_name]) + 1;
                        ndst++;
                }
        }

        /* Append room for core symbols at end of core part. */
        info->symoffs = ALIGN(mod_mem_data->size, symsect->sh_addralign ?: 1);
        info->stroffs = mod_mem_data->size = info->symoffs + ndst * sizeof(Elf_Sym);
        mod_mem_data->size += strtab_size;
        /* Note add_kallsyms() computes strtab_size as core_typeoffs - stroffs */
        info->core_typeoffs = mod_mem_data->size;
        mod_mem_data->size += ndst * sizeof(char);

        /* Put string table section at end of init part of module. */
        strsect->sh_flags |= SHF_ALLOC;
        strsect->sh_entsize = module_get_offset_and_type(mod, MOD_INIT_DATA,
                                                         strsect, info->index.str);
        pr_debug("\t%s\n", info->secstrings + strsect->sh_name);

        /* We'll tack temporary mod_kallsyms on the end. */
        mod_mem_init_data->size = ALIGN(mod_mem_init_data->size,
                                        __alignof__(struct mod_kallsyms));
        info->mod_kallsyms_init_off = mod_mem_init_data->size;

        mod_mem_init_data->size += sizeof(struct mod_kallsyms);
        info->init_typeoffs = mod_mem_init_data->size;
        mod_mem_init_data->size += nsrc * sizeof(char);
}

/*
 * We use the full symtab and strtab which layout_symtab arranged to
 * be appended to the init section.  Later we switch to the cut-down
 * core-only ones.
 */
void add_kallsyms(struct module *mod, const struct load_info *info)
{
        unsigned int i, ndst;
        const Elf_Sym *src;
        Elf_Sym *dst;
        char *s;
        Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
        unsigned long strtab_size;
        void *data_base = mod->mem[MOD_DATA].base;
        void *init_data_base = mod->mem[MOD_INIT_DATA].base;
        struct mod_kallsyms *kallsyms;

        kallsyms = init_data_base + info->mod_kallsyms_init_off;

        kallsyms->symtab = (void *)symsec->sh_addr;
        kallsyms->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
        /* Make sure we get permanent strtab: don't use info->strtab. */
        kallsyms->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
        kallsyms->typetab = init_data_base + info->init_typeoffs;

        /*
         * Now populate the cut down core kallsyms for after init
         * and set types up while we still have access to sections.
         */
        mod->core_kallsyms.symtab = dst = data_base + info->symoffs;
        mod->core_kallsyms.strtab = s = data_base + info->stroffs;
        mod->core_kallsyms.typetab = data_base + info->core_typeoffs;
        strtab_size = info->core_typeoffs - info->stroffs;
        src = kallsyms->symtab;
        for (ndst = i = 0; i < kallsyms->num_symtab; i++) {
                kallsyms->typetab[i] = elf_type(src + i, info);
                if (i == 0 || is_livepatch_module(mod) ||
                    is_core_symbol(src + i, info->sechdrs, info->hdr->e_shnum,
                                   info->index.pcpu)) {
                        ssize_t ret;

                        mod->core_kallsyms.typetab[ndst] =
                                kallsyms->typetab[i];
                        dst[ndst] = src[i];
                        dst[ndst++].st_name = s - mod->core_kallsyms.strtab;
                        ret = strscpy(s, &kallsyms->strtab[src[i].st_name],
                                      strtab_size);
                        if (ret < 0)
                                break;
                        s += ret + 1;
                        strtab_size -= ret + 1;
                }
        }

        /* Set up to point into init section. */
        rcu_assign_pointer(mod->kallsyms, kallsyms);
        mod->core_kallsyms.num_symtab = ndst;
}

#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID)
void init_build_id(struct module *mod, const struct load_info *info)
{
        const Elf_Shdr *sechdr;
        unsigned int i;

        for (i = 0; i < info->hdr->e_shnum; i++) {
                sechdr = &info->sechdrs[i];
                if (!sect_empty(sechdr) && sechdr->sh_type == SHT_NOTE &&
                    !build_id_parse_buf((void *)sechdr->sh_addr, mod->build_id,
                                        sechdr->sh_size))
                        break;
        }
}
#else
void init_build_id(struct module *mod, const struct load_info *info)
{
}
#endif

static const char *kallsyms_symbol_name(struct mod_kallsyms *kallsyms, unsigned int symnum)
{
        return kallsyms->strtab + kallsyms->symtab[symnum].st_name;
}

/*
 * Given a module and address, find the corresponding symbol and return its name
 * while providing its size and offset if needed.
 */
static const char *find_kallsyms_symbol(struct module *mod,
                                        unsigned long addr,
                                        unsigned long *size,
                                        unsigned long *offset)
{
        unsigned int i, best = 0;
        unsigned long nextval, bestval;
        struct mod_kallsyms *kallsyms = rcu_dereference(mod->kallsyms);
        struct module_memory *mod_mem;

        /* At worse, next value is at end of module */
        if (within_module_init(addr, mod))
                mod_mem = &mod->mem[MOD_INIT_TEXT];
        else
                mod_mem = &mod->mem[MOD_TEXT];

        nextval = (unsigned long)mod_mem->base + mod_mem->size;

        bestval = kallsyms_symbol_value(&kallsyms->symtab[best]);

        /*
         * Scan for closest preceding symbol, and next symbol. (ELF
         * starts real symbols at 1).
         */
        for (i = 1; i < kallsyms->num_symtab; i++) {
                const Elf_Sym *sym = &kallsyms->symtab[i];
                unsigned long thisval = kallsyms_symbol_value(sym);

                if (sym->st_shndx == SHN_UNDEF)
                        continue;

                /*
                 * We ignore unnamed symbols: they're uninformative
                 * and inserted at a whim.
                 */
                if (*kallsyms_symbol_name(kallsyms, i) == '\0' ||
                    is_mapping_symbol(kallsyms_symbol_name(kallsyms, i)))
                        continue;

                if (thisval <= addr && thisval > bestval) {
                        best = i;
                        bestval = thisval;
                }
                if (thisval > addr && thisval < nextval)
                        nextval = thisval;
        }

        if (!best)
                return NULL;

        if (size)
                *size = nextval - bestval;
        if (offset)
                *offset = addr - bestval;

        return kallsyms_symbol_name(kallsyms, best);
}

void * __weak dereference_module_function_descriptor(struct module *mod,
                                                     void *ptr)
{
        return ptr;
}

/*
 * For kallsyms to ask for address resolution.  NULL means not found.  Careful
 * not to lock to avoid deadlock on oopses, RCU is enough.
 */
int module_address_lookup(unsigned long addr,
                          unsigned long *size,
                          unsigned long *offset,
                          char **modname,
                          const unsigned char **modbuildid,
                          char *namebuf)
{
        const char *sym;
        int ret = 0;
        struct module *mod;

        guard(rcu)();
        mod = __module_address(addr);
        if (mod) {
                if (modname)
                        *modname = mod->name;
                if (modbuildid) {
#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID)
                        *modbuildid = mod->build_id;
#else
                        *modbuildid = NULL;
#endif
                }

                sym = find_kallsyms_symbol(mod, addr, size, offset);

                if (sym)
                        ret = strscpy(namebuf, sym, KSYM_NAME_LEN);
        }
        return ret;
}

int lookup_module_symbol_name(unsigned long addr, char *symname)
{
        struct module *mod;

        guard(rcu)();
        list_for_each_entry_rcu(mod, &modules, list) {
                if (mod->state == MODULE_STATE_UNFORMED)
                        continue;
                if (within_module(addr, mod)) {
                        const char *sym;

                        sym = find_kallsyms_symbol(mod, addr, NULL, NULL);
                        if (!sym)
                                goto out;

                        strscpy(symname, sym, KSYM_NAME_LEN);
                        return 0;
                }
        }
out:
        return -ERANGE;
}

int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
                       char *name, char *module_name, int *exported)
{
        struct module *mod;

        guard(rcu)();
        list_for_each_entry_rcu(mod, &modules, list) {
                struct mod_kallsyms *kallsyms;

                if (mod->state == MODULE_STATE_UNFORMED)
                        continue;
                kallsyms = rcu_dereference(mod->kallsyms);
                if (symnum < kallsyms->num_symtab) {
                        const Elf_Sym *sym = &kallsyms->symtab[symnum];

                        *value = kallsyms_symbol_value(sym);
                        *type = kallsyms->typetab[symnum];
                        strscpy(name, kallsyms_symbol_name(kallsyms, symnum), KSYM_NAME_LEN);
                        strscpy(module_name, mod->name, MODULE_NAME_LEN);
                        *exported = is_exported(name, *value, mod);
                        return 0;
                }
                symnum -= kallsyms->num_symtab;
        }
        return -ERANGE;
}

/* Given a module and name of symbol, find and return the symbol's value */
static unsigned long __find_kallsyms_symbol_value(struct module *mod, const char *name)
{
        unsigned int i;
        struct mod_kallsyms *kallsyms = rcu_dereference(mod->kallsyms);

        for (i = 0; i < kallsyms->num_symtab; i++) {
                const Elf_Sym *sym = &kallsyms->symtab[i];

                if (strcmp(name, kallsyms_symbol_name(kallsyms, i)) == 0 &&
                    sym->st_shndx != SHN_UNDEF)
                        return kallsyms_symbol_value(sym);
        }
        return 0;
}

static unsigned long __module_kallsyms_lookup_name(const char *name)
{
        struct module *mod;
        char *colon;

        colon = strnchr(name, MODULE_NAME_LEN, ':');
        if (colon) {
                mod = find_module_all(name, colon - name, false);
                if (mod)
                        return __find_kallsyms_symbol_value(mod, colon + 1);
                return 0;
        }

        list_for_each_entry_rcu(mod, &modules, list) {
                unsigned long ret;

                if (mod->state == MODULE_STATE_UNFORMED)
                        continue;
                ret = __find_kallsyms_symbol_value(mod, name);
                if (ret)
                        return ret;
        }
        return 0;
}

/* Look for this name: can be of form module:name. */
unsigned long module_kallsyms_lookup_name(const char *name)
{
        /* Don't lock: we're in enough trouble already. */
        guard(rcu)();
        return __module_kallsyms_lookup_name(name);
}

unsigned long find_kallsyms_symbol_value(struct module *mod, const char *name)
{
        guard(rcu)();
        return __find_kallsyms_symbol_value(mod, name);
}

int module_kallsyms_on_each_symbol(const char *modname,
                                   int (*fn)(void *, const char *, unsigned long),
                                   void *data)
{
        struct module *mod;
        unsigned int i;
        int ret = 0;

        mutex_lock(&module_mutex);
        list_for_each_entry(mod, &modules, list) {
                struct mod_kallsyms *kallsyms;

                if (mod->state == MODULE_STATE_UNFORMED)
                        continue;

                if (modname && strcmp(modname, mod->name))
                        continue;

                kallsyms = rcu_dereference_check(mod->kallsyms,
                                                 lockdep_is_held(&module_mutex));

                for (i = 0; i < kallsyms->num_symtab; i++) {
                        const Elf_Sym *sym = &kallsyms->symtab[i];

                        if (sym->st_shndx == SHN_UNDEF)
                                continue;

                        ret = fn(data, kallsyms_symbol_name(kallsyms, i),
                                 kallsyms_symbol_value(sym));
                        if (ret != 0)
                                goto out;
                }

                /*
                 * The given module is found, the subsequent modules do not
                 * need to be compared.
                 */
                if (modname)
                        break;
        }
out:
        mutex_unlock(&module_mutex);
        return ret;
}










































































































































  235 
  235 




   16 

















  220 























  220 




  220 





























  258 
   16 





























   16 









   16 













  250 
















   16 



















   16 














   16 


   16 







   16 



















   16 



   15 

















   16 



   16 



























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
// SPDX-License-Identifier: GPL-2.0-or-later

#include <linux/dma-resv.h>
#include <linux/dma-fence-chain.h>
#include <linux/export.h>

#include <drm/drm_atomic_state_helper.h>
#include <drm/drm_atomic_uapi.h>
#include <drm/drm_framebuffer.h>
#include <drm/drm_gem.h>
#include <drm/drm_gem_atomic_helper.h>
#include <drm/drm_gem_framebuffer_helper.h>
#include <drm/drm_simple_kms_helper.h>

#include "drm_internal.h"

/**
 * DOC: overview
 *
 * The GEM atomic helpers library implements generic atomic-commit
 * functions for drivers that use GEM objects. Currently, it provides
 * synchronization helpers, and plane state and framebuffer BO mappings
 * for planes with shadow buffers.
 *
 * Before scanout, a plane's framebuffer needs to be synchronized with
 * possible writers that draw into the framebuffer. All drivers should
 * call drm_gem_plane_helper_prepare_fb() from their implementation of
 * struct &drm_plane_helper.prepare_fb . It sets the plane's fence from
 * the framebuffer so that the DRM core can synchronize access automatically.
 * drm_gem_plane_helper_prepare_fb() can also be used directly as
 * implementation of prepare_fb.
 *
 * .. code-block:: c
 *
 *        #include <drm/drm_gem_atomic_helper.h>
 *
 *        struct drm_plane_helper_funcs driver_plane_helper_funcs = {
 *                ...,
 *                . prepare_fb = drm_gem_plane_helper_prepare_fb,
 *        };
 *
 * A driver using a shadow buffer copies the content of the shadow buffers
 * into the HW's framebuffer memory during an atomic update. This requires
 * a mapping of the shadow buffer into kernel address space. The mappings
 * cannot be established by commit-tail functions, such as atomic_update,
 * as this would violate locking rules around dma_buf_vmap().
 *
 * The helpers for shadow-buffered planes establish and release mappings,
 * and provide struct drm_shadow_plane_state, which stores the plane's mapping
 * for commit-tail functions.
 *
 * Shadow-buffered planes can easily be enabled by using the provided macros
 * %DRM_GEM_SHADOW_PLANE_FUNCS and %DRM_GEM_SHADOW_PLANE_HELPER_FUNCS.
 * These macros set up the plane and plane-helper callbacks to point to the
 * shadow-buffer helpers.
 *
 * .. code-block:: c
 *
 *        #include <drm/drm_gem_atomic_helper.h>
 *
 *        struct drm_plane_funcs driver_plane_funcs = {
 *                ...,
 *                DRM_GEM_SHADOW_PLANE_FUNCS,
 *        };
 *
 *        struct drm_plane_helper_funcs driver_plane_helper_funcs = {
 *                ...,
 *                DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
 *        };
 *
 * In the driver's atomic-update function, shadow-buffer mappings are available
 * from the plane state. Use to_drm_shadow_plane_state() to upcast from
 * struct drm_plane_state.
 *
 * .. code-block:: c
 *
 *        void driver_plane_atomic_update(struct drm_plane *plane,
 *                                        struct drm_plane_state *old_plane_state)
 *        {
 *                struct drm_plane_state *plane_state = plane->state;
 *                struct drm_shadow_plane_state *shadow_plane_state =
 *                        to_drm_shadow_plane_state(plane_state);
 *
 *                // access shadow buffer via shadow_plane_state->map
 *        }
 *
 * A mapping address for each of the framebuffer's buffer object is stored in
 * struct &drm_shadow_plane_state.map. The mappings are valid while the state
 * is being used.
 *
 * Drivers that use struct drm_simple_display_pipe can use
 * %DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS to initialize the rsp
 * callbacks. Access to shadow-buffer mappings is similar to regular
 * atomic_update.
 *
 * .. code-block:: c
 *
 *        struct drm_simple_display_pipe_funcs driver_pipe_funcs = {
 *                ...,
 *                DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS,
 *        };
 *
 *        void driver_pipe_enable(struct drm_simple_display_pipe *pipe,
 *                                struct drm_crtc_state *crtc_state,
 *                                struct drm_plane_state *plane_state)
 *        {
 *                struct drm_shadow_plane_state *shadow_plane_state =
 *                        to_drm_shadow_plane_state(plane_state);
 *
 *                // access shadow buffer via shadow_plane_state->map
 *        }
 */

/*
 * Plane Helpers
 */

/**
 * drm_gem_plane_helper_prepare_fb() - Prepare a GEM backed framebuffer
 * @plane: Plane
 * @state: Plane state the fence will be attached to
 *
 * This function extracts the exclusive fence from &drm_gem_object.resv and
 * attaches it to plane state for the atomic helper to wait on. This is
 * necessary to correctly implement implicit synchronization for any buffers
 * shared as a struct &dma_buf. This function can be used as the
 * &drm_plane_helper_funcs.prepare_fb callback.
 *
 * There is no need for &drm_plane_helper_funcs.cleanup_fb hook for simple
 * GEM based framebuffer drivers which have their buffers always pinned in
 * memory.
 *
 * This function is the default implementation for GEM drivers of
 * &drm_plane_helper_funcs.prepare_fb if no callback is provided.
 */
int drm_gem_plane_helper_prepare_fb(struct drm_plane *plane,
                                    struct drm_plane_state *state)
{
        struct dma_fence *fence = dma_fence_get(state->fence);
        enum dma_resv_usage usage;
        size_t i;
        int ret;

        if (!state->fb)
                return 0;

        /*
         * Only add the kernel fences here if there is already a fence set via
         * explicit fencing interfaces on the atomic ioctl.
         *
         * This way explicit fencing can be used to overrule implicit fencing,
         * which is important to make explicit fencing use-cases work: One
         * example is using one buffer for 2 screens with different refresh
         * rates. Implicit fencing will clamp rendering to the refresh rate of
         * the slower screen, whereas explicit fence allows 2 independent
         * render and display loops on a single buffer. If a driver allows
         * obeys both implicit and explicit fences for plane updates, then it
         * will break all the benefits of explicit fencing.
         */
        usage = fence ? DMA_RESV_USAGE_KERNEL : DMA_RESV_USAGE_WRITE;

        for (i = 0; i < state->fb->format->num_planes; ++i) {
                struct drm_gem_object *obj = drm_gem_fb_get_obj(state->fb, i);
                struct dma_fence *new;

                if (!obj) {
                        ret = -EINVAL;
                        goto error;
                }

                ret = dma_resv_get_singleton(obj->resv, usage, &new);
                if (ret)
                        goto error;

                if (new && fence) {
                        struct dma_fence_chain *chain = dma_fence_chain_alloc();

                        if (!chain) {
                                ret = -ENOMEM;
                                goto error;
                        }

                        dma_fence_chain_init(chain, fence, new, 1);
                        fence = &chain->base;

                } else if (new) {
                        fence = new;
                }
        }

        dma_fence_put(state->fence);
        state->fence = fence;
        return 0;

error:
        dma_fence_put(fence);
        return ret;
}
EXPORT_SYMBOL_GPL(drm_gem_plane_helper_prepare_fb);

/*
 * Shadow-buffered Planes
 */

/**
 * __drm_gem_duplicate_shadow_plane_state - duplicates shadow-buffered plane state
 * @plane: the plane
 * @new_shadow_plane_state: the new shadow-buffered plane state
 *
 * This function duplicates shadow-buffered plane state. This is helpful for drivers
 * that subclass struct drm_shadow_plane_state.
 *
 * The function does not duplicate existing mappings of the shadow buffers.
 * Mappings are maintained during the atomic commit by the plane's prepare_fb
 * and cleanup_fb helpers. See drm_gem_prepare_shadow_fb() and drm_gem_cleanup_shadow_fb()
 * for corresponding helpers.
 */
void
__drm_gem_duplicate_shadow_plane_state(struct drm_plane *plane,
                                       struct drm_shadow_plane_state *new_shadow_plane_state)
{
        struct drm_plane_state *plane_state = plane->state;
        struct drm_shadow_plane_state *shadow_plane_state =
                to_drm_shadow_plane_state(plane_state);

        __drm_atomic_helper_plane_duplicate_state(plane, &new_shadow_plane_state->base);

        drm_format_conv_state_copy(&new_shadow_plane_state->fmtcnv_state,
                                   &shadow_plane_state->fmtcnv_state);
}
EXPORT_SYMBOL(__drm_gem_duplicate_shadow_plane_state);

/**
 * drm_gem_duplicate_shadow_plane_state - duplicates shadow-buffered plane state
 * @plane: the plane
 *
 * This function implements struct &drm_plane_funcs.atomic_duplicate_state for
 * shadow-buffered planes. It assumes the existing state to be of type
 * struct drm_shadow_plane_state and it allocates the new state to be of this
 * type.
 *
 * The function does not duplicate existing mappings of the shadow buffers.
 * Mappings are maintained during the atomic commit by the plane's prepare_fb
 * and cleanup_fb helpers. See drm_gem_prepare_shadow_fb() and drm_gem_cleanup_shadow_fb()
 * for corresponding helpers.
 *
 * Returns:
 * A pointer to a new plane state on success, or NULL otherwise.
 */
struct drm_plane_state *
drm_gem_duplicate_shadow_plane_state(struct drm_plane *plane)
{
        struct drm_plane_state *plane_state = plane->state;
        struct drm_shadow_plane_state *new_shadow_plane_state;

        if (!plane_state)
                return NULL;

        new_shadow_plane_state = kzalloc(sizeof(*new_shadow_plane_state), GFP_KERNEL);
        if (!new_shadow_plane_state)
                return NULL;
        __drm_gem_duplicate_shadow_plane_state(plane, new_shadow_plane_state);

        return &new_shadow_plane_state->base;
}
EXPORT_SYMBOL(drm_gem_duplicate_shadow_plane_state);

/**
 * __drm_gem_destroy_shadow_plane_state - cleans up shadow-buffered plane state
 * @shadow_plane_state: the shadow-buffered plane state
 *
 * This function cleans up shadow-buffered plane state. Helpful for drivers that
 * subclass struct drm_shadow_plane_state.
 */
void __drm_gem_destroy_shadow_plane_state(struct drm_shadow_plane_state *shadow_plane_state)
{
        drm_format_conv_state_release(&shadow_plane_state->fmtcnv_state);
        __drm_atomic_helper_plane_destroy_state(&shadow_plane_state->base);
}
EXPORT_SYMBOL(__drm_gem_destroy_shadow_plane_state);

/**
 * drm_gem_destroy_shadow_plane_state - deletes shadow-buffered plane state
 * @plane: the plane
 * @plane_state: the plane state of type struct drm_shadow_plane_state
 *
 * This function implements struct &drm_plane_funcs.atomic_destroy_state
 * for shadow-buffered planes. It expects that mappings of shadow buffers
 * have been released already.
 */
void drm_gem_destroy_shadow_plane_state(struct drm_plane *plane,
                                        struct drm_plane_state *plane_state)
{
        struct drm_shadow_plane_state *shadow_plane_state =
                to_drm_shadow_plane_state(plane_state);

        __drm_gem_destroy_shadow_plane_state(shadow_plane_state);
        kfree(shadow_plane_state);
}
EXPORT_SYMBOL(drm_gem_destroy_shadow_plane_state);

/**
 * __drm_gem_reset_shadow_plane - resets a shadow-buffered plane
 * @plane: the plane
 * @shadow_plane_state: the shadow-buffered plane state
 *
 * This function resets state for shadow-buffered planes. Helpful
 * for drivers that subclass struct drm_shadow_plane_state.
 */
void __drm_gem_reset_shadow_plane(struct drm_plane *plane,
                                  struct drm_shadow_plane_state *shadow_plane_state)
{
        __drm_atomic_helper_plane_reset(plane, &shadow_plane_state->base);
        drm_format_conv_state_init(&shadow_plane_state->fmtcnv_state);
}
EXPORT_SYMBOL(__drm_gem_reset_shadow_plane);

/**
 * drm_gem_reset_shadow_plane - resets a shadow-buffered plane
 * @plane: the plane
 *
 * This function implements struct &drm_plane_funcs.reset_plane for
 * shadow-buffered planes. It assumes the current plane state to be
 * of type struct drm_shadow_plane and it allocates the new state of
 * this type.
 */
void drm_gem_reset_shadow_plane(struct drm_plane *plane)
{
        struct drm_shadow_plane_state *shadow_plane_state;

        if (plane->state) {
                drm_gem_destroy_shadow_plane_state(plane, plane->state);
                plane->state = NULL; /* must be set to NULL here */
        }

        shadow_plane_state = kzalloc(sizeof(*shadow_plane_state), GFP_KERNEL);
        if (!shadow_plane_state)
                return;
        __drm_gem_reset_shadow_plane(plane, shadow_plane_state);
}
EXPORT_SYMBOL(drm_gem_reset_shadow_plane);

/**
 * drm_gem_begin_shadow_fb_access - prepares shadow framebuffers for CPU access
 * @plane: the plane
 * @plane_state: the plane state of type struct drm_shadow_plane_state
 *
 * This function implements struct &drm_plane_helper_funcs.begin_fb_access. It
 * maps all buffer objects of the plane's framebuffer into kernel address
 * space and stores them in struct &drm_shadow_plane_state.map. The first data
 * bytes are available in struct &drm_shadow_plane_state.data.
 *
 * See drm_gem_end_shadow_fb_access() for cleanup.
 *
 * Returns:
 * 0 on success, or a negative errno code otherwise.
 */
int drm_gem_begin_shadow_fb_access(struct drm_plane *plane, struct drm_plane_state *plane_state)
{
        struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
        struct drm_framebuffer *fb = plane_state->fb;

        if (!fb)
                return 0;

        return drm_gem_fb_vmap(fb, shadow_plane_state->map, shadow_plane_state->data);
}
EXPORT_SYMBOL(drm_gem_begin_shadow_fb_access);

/**
 * drm_gem_end_shadow_fb_access - releases shadow framebuffers from CPU access
 * @plane: the plane
 * @plane_state: the plane state of type struct drm_shadow_plane_state
 *
 * This function implements struct &drm_plane_helper_funcs.end_fb_access. It
 * undoes all effects of drm_gem_begin_shadow_fb_access() in reverse order.
 *
 * See drm_gem_begin_shadow_fb_access() for more information.
 */
void drm_gem_end_shadow_fb_access(struct drm_plane *plane, struct drm_plane_state *plane_state)
{
        struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
        struct drm_framebuffer *fb = plane_state->fb;

        if (!fb)
                return;

        drm_gem_fb_vunmap(fb, shadow_plane_state->map);
}
EXPORT_SYMBOL(drm_gem_end_shadow_fb_access);

/**
 * drm_gem_simple_kms_begin_shadow_fb_access - prepares shadow framebuffers for CPU access
 * @pipe: the simple display pipe
 * @plane_state: the plane state of type struct drm_shadow_plane_state
 *
 * This function implements struct drm_simple_display_funcs.begin_fb_access.
 *
 * See drm_gem_begin_shadow_fb_access() for details and
 * drm_gem_simple_kms_cleanup_shadow_fb() for cleanup.
 *
 * Returns:
 * 0 on success, or a negative errno code otherwise.
 */
int drm_gem_simple_kms_begin_shadow_fb_access(struct drm_simple_display_pipe *pipe,
                                              struct drm_plane_state *plane_state)
{
        return drm_gem_begin_shadow_fb_access(&pipe->plane, plane_state);
}
EXPORT_SYMBOL(drm_gem_simple_kms_begin_shadow_fb_access);

/**
 * drm_gem_simple_kms_end_shadow_fb_access - releases shadow framebuffers from CPU access
 * @pipe: the simple display pipe
 * @plane_state: the plane state of type struct drm_shadow_plane_state
 *
 * This function implements struct drm_simple_display_funcs.end_fb_access.
 * It undoes all effects of drm_gem_simple_kms_begin_shadow_fb_access() in
 * reverse order.
 *
 * See drm_gem_simple_kms_begin_shadow_fb_access().
 */
void drm_gem_simple_kms_end_shadow_fb_access(struct drm_simple_display_pipe *pipe,
                                             struct drm_plane_state *plane_state)
{
        drm_gem_end_shadow_fb_access(&pipe->plane, plane_state);
}
EXPORT_SYMBOL(drm_gem_simple_kms_end_shadow_fb_access);

/**
 * drm_gem_simple_kms_reset_shadow_plane - resets a shadow-buffered plane
 * @pipe: the simple display pipe
 *
 * This function implements struct drm_simple_display_funcs.reset_plane
 * for shadow-buffered planes.
 */
void drm_gem_simple_kms_reset_shadow_plane(struct drm_simple_display_pipe *pipe)
{
        drm_gem_reset_shadow_plane(&pipe->plane);
}
EXPORT_SYMBOL(drm_gem_simple_kms_reset_shadow_plane);

/**
 * drm_gem_simple_kms_duplicate_shadow_plane_state - duplicates shadow-buffered plane state
 * @pipe: the simple display pipe
 *
 * This function implements struct drm_simple_display_funcs.duplicate_plane_state
 * for shadow-buffered planes. It does not duplicate existing mappings of the shadow
 * buffers. Mappings are maintained during the atomic commit by the plane's prepare_fb
 * and cleanup_fb helpers.
 *
 * Returns:
 * A pointer to a new plane state on success, or NULL otherwise.
 */
struct drm_plane_state *
drm_gem_simple_kms_duplicate_shadow_plane_state(struct drm_simple_display_pipe *pipe)
{
        return drm_gem_duplicate_shadow_plane_state(&pipe->plane);
}
EXPORT_SYMBOL(drm_gem_simple_kms_duplicate_shadow_plane_state);

/**
 * drm_gem_simple_kms_destroy_shadow_plane_state - resets shadow-buffered plane state
 * @pipe: the simple display pipe
 * @plane_state: the plane state of type struct drm_shadow_plane_state
 *
 * This function implements struct drm_simple_display_funcs.destroy_plane_state
 * for shadow-buffered planes. It expects that mappings of shadow buffers
 * have been released already.
 */
void drm_gem_simple_kms_destroy_shadow_plane_state(struct drm_simple_display_pipe *pipe,
                                                   struct drm_plane_state *plane_state)
{
        drm_gem_destroy_shadow_plane_state(&pipe->plane, plane_state);
}
EXPORT_SYMBOL(drm_gem_simple_kms_destroy_shadow_plane_state);



























































































































































































































































































































































































  370 





  315 




































































































    2 
























   28 
   54 
   48 
   48 
   30 















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (c) 1999-2002 Vojtech Pavlik
 */
#ifndef _INPUT_H
#define _INPUT_H

#include <linux/time.h>
#include <linux/list.h>
#include <uapi/linux/input.h>
/* Implementation details, userspace should not care about these */
#define ABS_MT_FIRST                ABS_MT_TOUCH_MAJOR
#define ABS_MT_LAST                ABS_MT_TOOL_Y

/*
 * In-kernel definitions.
 */

#include <linux/device.h>
#include <linux/fs.h>
#include <linux/timer.h>
#include <linux/mod_devicetable.h>

struct input_dev_poller;

/**
 * struct input_value - input value representation
 * @type: type of value (EV_KEY, EV_ABS, etc)
 * @code: the value code
 * @value: the value
 */
struct input_value {
        __u16 type;
        __u16 code;
        __s32 value;
};

enum input_clock_type {
        INPUT_CLK_REAL = 0,
        INPUT_CLK_MONO,
        INPUT_CLK_BOOT,
        INPUT_CLK_MAX
};

/**
 * struct input_dev - represents an input device
 * @name: name of the device
 * @phys: physical path to the device in the system hierarchy
 * @uniq: unique identification code for the device (if device has it)
 * @id: id of the device (struct input_id)
 * @propbit: bitmap of device properties and quirks
 * @evbit: bitmap of types of events supported by the device (EV_KEY,
 *        EV_REL, etc.)
 * @keybit: bitmap of keys/buttons this device has
 * @relbit: bitmap of relative axes for the device
 * @absbit: bitmap of absolute axes for the device
 * @mscbit: bitmap of miscellaneous events supported by the device
 * @ledbit: bitmap of leds present on the device
 * @sndbit: bitmap of sound effects supported by the device
 * @ffbit: bitmap of force feedback effects supported by the device
 * @swbit: bitmap of switches present on the device
 * @hint_events_per_packet: average number of events generated by the
 *        device in a packet (between EV_SYN/SYN_REPORT events). Used by
 *        event handlers to estimate size of the buffer needed to hold
 *        events.
 * @keycodemax: size of keycode table
 * @keycodesize: size of elements in keycode table
 * @keycode: map of scancodes to keycodes for this device
 * @getkeycode: optional legacy method to retrieve current keymap.
 * @setkeycode: optional method to alter current keymap, used to implement
 *        sparse keymaps. If not supplied default mechanism will be used.
 *        The method is being called while holding event_lock and thus must
 *        not sleep
 * @ff: force feedback structure associated with the device if device
 *        supports force feedback effects
 * @poller: poller structure associated with the device if device is
 *        set up to use polling mode
 * @repeat_key: stores key code of the last key pressed; used to implement
 *        software autorepeat
 * @timer: timer for software autorepeat
 * @rep: current values for autorepeat parameters (delay, rate)
 * @mt: pointer to multitouch state
 * @absinfo: array of &struct input_absinfo elements holding information
 *        about absolute axes (current value, min, max, flat, fuzz,
 *        resolution)
 * @key: reflects current state of device's keys/buttons
 * @led: reflects current state of device's LEDs
 * @snd: reflects current state of sound effects
 * @sw: reflects current state of device's switches
 * @open: this method is called when the very first user calls
 *        input_open_device(). The driver must prepare the device
 *        to start generating events (start polling thread,
 *        request an IRQ, submit URB, etc.). The meaning of open() is
 *        to start providing events to the input core.
 * @close: this method is called when the very last user calls
 *        input_close_device(). The meaning of close() is to stop
 *        providing events to the input core.
 * @flush: purges the device. Most commonly used to get rid of force
 *        feedback effects loaded into the device when disconnecting
 *        from it
 * @event: event handler for events sent _to_ the device, like EV_LED
 *        or EV_SND. The device is expected to carry out the requested
 *        action (turn on a LED, play sound, etc.) The call is protected
 *        by @event_lock and must not sleep
 * @grab: input handle that currently has the device grabbed (via
 *        EVIOCGRAB ioctl). When a handle grabs a device it becomes sole
 *        recipient for all input events coming from the device
 * @event_lock: this spinlock is taken when input core receives
 *        and processes a new event for the device (in input_event()).
 *        Code that accesses and/or modifies parameters of a device
 *        (such as keymap or absmin, absmax, absfuzz, etc.) after device
 *        has been registered with input core must take this lock.
 * @mutex: serializes calls to open(), close() and flush() methods
 * @users: stores number of users (input handlers) that opened this
 *        device. It is used by input_open_device() and input_close_device()
 *        to make sure that dev->open() is only called when the first
 *        user opens device and dev->close() is called when the very
 *        last user closes the device
 * @going_away: marks devices that are in a middle of unregistering and
 *        causes input_open_device*() fail with -ENODEV.
 * @dev: driver model's view of this device
 * @h_list: list of input handles associated with the device. When
 *        accessing the list dev->mutex must be held
 * @node: used to place the device onto input_dev_list
 * @num_vals: number of values queued in the current frame
 * @max_vals: maximum number of values queued in a frame
 * @vals: array of values queued in the current frame
 * @devres_managed: indicates that devices is managed with devres framework
 *        and needs not be explicitly unregistered or freed.
 * @timestamp: storage for a timestamp set by input_set_timestamp called
 *  by a driver
 * @inhibited: indicates that the input device is inhibited. If that is
 * the case then input core ignores any events generated by the device.
 * Device's close() is called when it is being inhibited and its open()
 * is called when it is being uninhibited.
 */
struct input_dev {
        const char *name;
        const char *phys;
        const char *uniq;
        struct input_id id;

        unsigned long propbit[BITS_TO_LONGS(INPUT_PROP_CNT)];

        unsigned long evbit[BITS_TO_LONGS(EV_CNT)];
        unsigned long keybit[BITS_TO_LONGS(KEY_CNT)];
        unsigned long relbit[BITS_TO_LONGS(REL_CNT)];
        unsigned long absbit[BITS_TO_LONGS(ABS_CNT)];
        unsigned long mscbit[BITS_TO_LONGS(MSC_CNT)];
        unsigned long ledbit[BITS_TO_LONGS(LED_CNT)];
        unsigned long sndbit[BITS_TO_LONGS(SND_CNT)];
        unsigned long ffbit[BITS_TO_LONGS(FF_CNT)];
        unsigned long swbit[BITS_TO_LONGS(SW_CNT)];

        unsigned int hint_events_per_packet;

        unsigned int keycodemax;
        unsigned int keycodesize;
        void *keycode;

        int (*setkeycode)(struct input_dev *dev,
                          const struct input_keymap_entry *ke,
                          unsigned int *old_keycode);
        int (*getkeycode)(struct input_dev *dev,
                          struct input_keymap_entry *ke);

        struct ff_device *ff;

        struct input_dev_poller *poller;

        unsigned int repeat_key;
        struct timer_list timer;

        int rep[REP_CNT];

        struct input_mt *mt;

        struct input_absinfo *absinfo;

        unsigned long key[BITS_TO_LONGS(KEY_CNT)];
        unsigned long led[BITS_TO_LONGS(LED_CNT)];
        unsigned long snd[BITS_TO_LONGS(SND_CNT)];
        unsigned long sw[BITS_TO_LONGS(SW_CNT)];

        int (*open)(struct input_dev *dev);
        void (*close)(struct input_dev *dev);
        int (*flush)(struct input_dev *dev, struct file *file);
        int (*event)(struct input_dev *dev, unsigned int type, unsigned int code, int value);

        struct input_handle __rcu *grab;

        spinlock_t event_lock;
        struct mutex mutex;

        unsigned int users;
        bool going_away;

        struct device dev;

        struct list_head        h_list;
        struct list_head        node;

        unsigned int num_vals;
        unsigned int max_vals;
        struct input_value *vals;

        bool devres_managed;

        ktime_t timestamp[INPUT_CLK_MAX];

        bool inhibited;
};
#define to_input_dev(d) container_of(d, struct input_dev, dev)

/*
 * Verify that we are in sync with input_device_id mod_devicetable.h #defines
 */

#if EV_MAX != INPUT_DEVICE_ID_EV_MAX
#error "EV_MAX and INPUT_DEVICE_ID_EV_MAX do not match"
#endif

#if KEY_MIN_INTERESTING != INPUT_DEVICE_ID_KEY_MIN_INTERESTING
#error "KEY_MIN_INTERESTING and INPUT_DEVICE_ID_KEY_MIN_INTERESTING do not match"
#endif

#if KEY_MAX != INPUT_DEVICE_ID_KEY_MAX
#error "KEY_MAX and INPUT_DEVICE_ID_KEY_MAX do not match"
#endif

#if REL_MAX != INPUT_DEVICE_ID_REL_MAX
#error "REL_MAX and INPUT_DEVICE_ID_REL_MAX do not match"
#endif

#if ABS_MAX != INPUT_DEVICE_ID_ABS_MAX
#error "ABS_MAX and INPUT_DEVICE_ID_ABS_MAX do not match"
#endif

#if MSC_MAX != INPUT_DEVICE_ID_MSC_MAX
#error "MSC_MAX and INPUT_DEVICE_ID_MSC_MAX do not match"
#endif

#if LED_MAX != INPUT_DEVICE_ID_LED_MAX
#error "LED_MAX and INPUT_DEVICE_ID_LED_MAX do not match"
#endif

#if SND_MAX != INPUT_DEVICE_ID_SND_MAX
#error "SND_MAX and INPUT_DEVICE_ID_SND_MAX do not match"
#endif

#if FF_MAX != INPUT_DEVICE_ID_FF_MAX
#error "FF_MAX and INPUT_DEVICE_ID_FF_MAX do not match"
#endif

#if SW_MAX != INPUT_DEVICE_ID_SW_MAX
#error "SW_MAX and INPUT_DEVICE_ID_SW_MAX do not match"
#endif

#if INPUT_PROP_MAX != INPUT_DEVICE_ID_PROP_MAX
#error "INPUT_PROP_MAX and INPUT_DEVICE_ID_PROP_MAX do not match"
#endif

#define INPUT_DEVICE_ID_MATCH_DEVICE \
        (INPUT_DEVICE_ID_MATCH_BUS | INPUT_DEVICE_ID_MATCH_VENDOR | INPUT_DEVICE_ID_MATCH_PRODUCT)
#define INPUT_DEVICE_ID_MATCH_DEVICE_AND_VERSION \
        (INPUT_DEVICE_ID_MATCH_DEVICE | INPUT_DEVICE_ID_MATCH_VERSION)

struct input_handle;

/**
 * struct input_handler - implements one of interfaces for input devices
 * @private: driver-specific data
 * @event: event handler. This method is being called by input core with
 *        interrupts disabled and dev->event_lock spinlock held and so
 *        it may not sleep
 * @events: event sequence handler. This method is being called by
 *        input core with interrupts disabled and dev->event_lock
 *        spinlock held and so it may not sleep. The method must return
 *        number of events passed to it.
 * @filter: similar to @event; separates normal event handlers from
 *        "filters".
 * @match: called after comparing device's id with handler's id_table
 *        to perform fine-grained matching between device and handler
 * @connect: called when attaching a handler to an input device
 * @disconnect: disconnects a handler from input device
 * @start: starts handler for given handle. This function is called by
 *        input core right after connect() method and also when a process
 *        that "grabbed" a device releases it
 * @passive_observer: set to %true by drivers only interested in observing
 *        data stream from devices if there are other users present. Such
 *        drivers will not result in starting underlying hardware device
 *        when input_open_device() is called for their handles
 * @legacy_minors: set to %true by drivers using legacy minor ranges
 * @minor: beginning of range of 32 legacy minors for devices this driver
 *        can provide
 * @name: name of the handler, to be shown in /proc/bus/input/handlers
 * @id_table: pointer to a table of input_device_ids this driver can
 *        handle
 * @h_list: list of input handles associated with the handler
 * @node: for placing the driver onto input_handler_list
 *
 * Input handlers attach to input devices and create input handles. There
 * are likely several handlers attached to any given input device at the
 * same time. All of them will get their copy of input event generated by
 * the device.
 *
 * The very same structure is used to implement input filters. Input core
 * allows filters to run first and will not pass event to regular handlers
 * if any of the filters indicate that the event should be filtered (by
 * returning %true from their filter() method).
 *
 * Note that input core serializes calls to connect() and disconnect()
 * methods.
 */
struct input_handler {

        void *private;

        void (*event)(struct input_handle *handle, unsigned int type, unsigned int code, int value);
        unsigned int (*events)(struct input_handle *handle,
                               struct input_value *vals, unsigned int count);
        bool (*filter)(struct input_handle *handle, unsigned int type, unsigned int code, int value);
        bool (*match)(struct input_handler *handler, struct input_dev *dev);
        int (*connect)(struct input_handler *handler, struct input_dev *dev, const struct input_device_id *id);
        void (*disconnect)(struct input_handle *handle);
        void (*start)(struct input_handle *handle);

        bool passive_observer;
        bool legacy_minors;
        int minor;
        const char *name;

        const struct input_device_id *id_table;

        struct list_head        h_list;
        struct list_head        node;
};

/**
 * struct input_handle - links input device with an input handler
 * @private: handler-specific data
 * @open: counter showing whether the handle is 'open', i.e. should deliver
 *        events from its device
 * @name: name given to the handle by handler that created it
 * @dev: input device the handle is attached to
 * @handler: handler that works with the device through this handle
 * @handle_events: event sequence handler. It is set up by the input core
 *        according to event handling method specified in the @handler. See
 *        input_handle_setup_event_handler().
 *        This method is being called by the input core with interrupts disabled
 *        and dev->event_lock spinlock held and so it may not sleep.
 * @d_node: used to put the handle on device's list of attached handles
 * @h_node: used to put the handle on handler's list of handles from which
 *        it gets events
 */
struct input_handle {
        void *private;

        int open;
        const char *name;

        struct input_dev *dev;
        struct input_handler *handler;

        unsigned int (*handle_events)(struct input_handle *handle,
                                      struct input_value *vals,
                                      unsigned int count);

        struct list_head        d_node;
        struct list_head        h_node;
};

struct input_dev __must_check *input_allocate_device(void);
struct input_dev __must_check *devm_input_allocate_device(struct device *);
void input_free_device(struct input_dev *dev);

static inline struct input_dev *input_get_device(struct input_dev *dev)
{
        return dev ? to_input_dev(get_device(&dev->dev)) : NULL;
}

static inline void input_put_device(struct input_dev *dev)
{
        if (dev)
                put_device(&dev->dev);
}

static inline void *input_get_drvdata(struct input_dev *dev)
{
        return dev_get_drvdata(&dev->dev);
}

static inline void input_set_drvdata(struct input_dev *dev, void *data)
{
        dev_set_drvdata(&dev->dev, data);
}

int __must_check input_register_device(struct input_dev *);
void input_unregister_device(struct input_dev *);

void input_reset_device(struct input_dev *);

int input_setup_polling(struct input_dev *dev,
                        void (*poll_fn)(struct input_dev *dev));
void input_set_poll_interval(struct input_dev *dev, unsigned int interval);
void input_set_min_poll_interval(struct input_dev *dev, unsigned int interval);
void input_set_max_poll_interval(struct input_dev *dev, unsigned int interval);
int input_get_poll_interval(struct input_dev *dev);

int __must_check input_register_handler(struct input_handler *);
void input_unregister_handler(struct input_handler *);

int __must_check input_get_new_minor(int legacy_base, unsigned int legacy_num,
                                     bool allow_dynamic);
void input_free_minor(unsigned int minor);

int input_handler_for_each_handle(struct input_handler *, void *data,
                                  int (*fn)(struct input_handle *, void *));

int input_register_handle(struct input_handle *);
void input_unregister_handle(struct input_handle *);

int input_grab_device(struct input_handle *);
void input_release_device(struct input_handle *);

int input_open_device(struct input_handle *);
void input_close_device(struct input_handle *);

int input_flush_device(struct input_handle *handle, struct file *file);

void input_set_timestamp(struct input_dev *dev, ktime_t timestamp);
ktime_t *input_get_timestamp(struct input_dev *dev);

void input_event(struct input_dev *dev, unsigned int type, unsigned int code, int value);
void input_inject_event(struct input_handle *handle, unsigned int type, unsigned int code, int value);

static inline void input_report_key(struct input_dev *dev, unsigned int code, int value)
{
        input_event(dev, EV_KEY, code, !!value);
}

static inline void input_report_rel(struct input_dev *dev, unsigned int code, int value)
{
        input_event(dev, EV_REL, code, value);
}

static inline void input_report_abs(struct input_dev *dev, unsigned int code, int value)
{
        input_event(dev, EV_ABS, code, value);
}

static inline void input_report_ff_status(struct input_dev *dev, unsigned int code, int value)
{
        input_event(dev, EV_FF_STATUS, code, value);
}

static inline void input_report_switch(struct input_dev *dev, unsigned int code, int value)
{
        input_event(dev, EV_SW, code, !!value);
}

static inline void input_sync(struct input_dev *dev)
{
        input_event(dev, EV_SYN, SYN_REPORT, 0);
}

static inline void input_mt_sync(struct input_dev *dev)
{
        input_event(dev, EV_SYN, SYN_MT_REPORT, 0);
}

void input_set_capability(struct input_dev *dev, unsigned int type, unsigned int code);

/**
 * input_set_events_per_packet - tell handlers about the driver event rate
 * @dev: the input device used by the driver
 * @n_events: the average number of events between calls to input_sync()
 *
 * If the event rate sent from a device is unusually large, use this
 * function to set the expected event rate. This will allow handlers
 * to set up an appropriate buffer size for the event stream, in order
 * to minimize information loss.
 */
static inline void input_set_events_per_packet(struct input_dev *dev, int n_events)
{
        dev->hint_events_per_packet = n_events;
}

void input_alloc_absinfo(struct input_dev *dev);
void input_set_abs_params(struct input_dev *dev, unsigned int axis,
                          int min, int max, int fuzz, int flat);
void input_copy_abs(struct input_dev *dst, unsigned int dst_axis,
                    const struct input_dev *src, unsigned int src_axis);

#define INPUT_GENERATE_ABS_ACCESSORS(_suffix, _item)                        \
static inline int input_abs_get_##_suffix(struct input_dev *dev,        \
                                          unsigned int axis)                \
{                                                                        \
        return dev->absinfo ? dev->absinfo[axis]._item : 0;                \
}                                                                        \
                                                                        \
static inline void input_abs_set_##_suffix(struct input_dev *dev,        \
                                           unsigned int axis, int val)        \
{                                                                        \
        input_alloc_absinfo(dev);                                        \
        if (dev->absinfo)                                                \
                dev->absinfo[axis]._item = val;                                \
}

INPUT_GENERATE_ABS_ACCESSORS(val, value)
INPUT_GENERATE_ABS_ACCESSORS(min, minimum)
INPUT_GENERATE_ABS_ACCESSORS(max, maximum)
INPUT_GENERATE_ABS_ACCESSORS(fuzz, fuzz)
INPUT_GENERATE_ABS_ACCESSORS(flat, flat)
INPUT_GENERATE_ABS_ACCESSORS(res, resolution)

int input_scancode_to_scalar(const struct input_keymap_entry *ke,
                             unsigned int *scancode);

int input_get_keycode(struct input_dev *dev, struct input_keymap_entry *ke);
int input_set_keycode(struct input_dev *dev,
                      const struct input_keymap_entry *ke);

bool input_match_device_id(const struct input_dev *dev,
                           const struct input_device_id *id);

void input_enable_softrepeat(struct input_dev *dev, int delay, int period);

bool input_device_enabled(struct input_dev *dev);

extern const struct class input_class;

/**
 * struct ff_device - force-feedback part of an input device
 * @upload: Called to upload an new effect into device
 * @erase: Called to erase an effect from device
 * @playback: Called to request device to start playing specified effect
 * @set_gain: Called to set specified gain
 * @set_autocenter: Called to auto-center device
 * @destroy: called by input core when parent input device is being
 *        destroyed
 * @private: driver-specific data, will be freed automatically
 * @ffbit: bitmap of force feedback capabilities truly supported by
 *        device (not emulated like ones in input_dev->ffbit)
 * @mutex: mutex for serializing access to the device
 * @max_effects: maximum number of effects supported by device
 * @effects: pointer to an array of effects currently loaded into device
 * @effect_owners: array of effect owners; when file handle owning
 *        an effect gets closed the effect is automatically erased
 *
 * Every force-feedback device must implement upload() and playback()
 * methods; erase() is optional. set_gain() and set_autocenter() need
 * only be implemented if driver sets up FF_GAIN and FF_AUTOCENTER
 * bits.
 *
 * Note that playback(), set_gain() and set_autocenter() are called with
 * dev->event_lock spinlock held and interrupts off and thus may not
 * sleep.
 */
struct ff_device {
        int (*upload)(struct input_dev *dev, struct ff_effect *effect,
                      struct ff_effect *old);
        int (*erase)(struct input_dev *dev, int effect_id);

        int (*playback)(struct input_dev *dev, int effect_id, int value);
        void (*set_gain)(struct input_dev *dev, u16 gain);
        void (*set_autocenter)(struct input_dev *dev, u16 magnitude);

        void (*destroy)(struct ff_device *);

        void *private;

        unsigned long ffbit[BITS_TO_LONGS(FF_CNT)];

        struct mutex mutex;

        int max_effects;
        struct ff_effect *effects;
        struct file *effect_owners[] __counted_by(max_effects);
};

int input_ff_create(struct input_dev *dev, unsigned int max_effects);
void input_ff_destroy(struct input_dev *dev);

int input_ff_event(struct input_dev *dev, unsigned int type, unsigned int code, int value);

int input_ff_upload(struct input_dev *dev, struct ff_effect *effect, struct file *file);
int input_ff_erase(struct input_dev *dev, int effect_id, struct file *file);
int input_ff_flush(struct input_dev *dev, struct file *file);

int input_ff_create_memless(struct input_dev *dev, void *data,
                int (*play_effect)(struct input_dev *, void *, struct ff_effect *));

#endif




























































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * AppArmor security module
 *
 * This file contains AppArmor basic permission sets definitions.
 *
 * Copyright 2017 Canonical Ltd.
 */

#ifndef __AA_PERM_H
#define __AA_PERM_H

#include <linux/fs.h>
#include "label.h"

#define AA_MAY_EXEC                MAY_EXEC
#define AA_MAY_WRITE                MAY_WRITE
#define AA_MAY_READ                MAY_READ
#define AA_MAY_APPEND                MAY_APPEND

#define AA_MAY_CREATE                0x0010
#define AA_MAY_DELETE                0x0020
#define AA_MAY_OPEN                0x0040
#define AA_MAY_RENAME                0x0080                /* pair */

#define AA_MAY_SETATTR                0x0100                /* meta write */
#define AA_MAY_GETATTR                0x0200                /* meta read */
#define AA_MAY_SETCRED                0x0400                /* security cred/attr */
#define AA_MAY_GETCRED                0x0800

#define AA_MAY_CHMOD                0x1000                /* pair */
#define AA_MAY_CHOWN                0x2000                /* pair */
#define AA_MAY_CHGRP                0x4000                /* pair */
#define AA_MAY_LOCK                0x8000                /* LINK_SUBSET overlaid */

#define AA_EXEC_MMAP                0x00010000
#define AA_MAY_MPROT                0x00020000        /* extend conditions */
#define AA_MAY_LINK                0x00040000        /* pair */
#define AA_MAY_SNAPSHOT                0x00080000        /* pair */

#define AA_MAY_DELEGATE
#define AA_CONT_MATCH                0x08000000

#define AA_MAY_STACK                0x10000000
#define AA_MAY_ONEXEC                0x20000000 /* either stack or change_profile */
#define AA_MAY_CHANGE_PROFILE        0x40000000
#define AA_MAY_CHANGEHAT        0x80000000

#define AA_LINK_SUBSET                AA_MAY_LOCK        /* overlaid */

#define AA_MAY_CREATE_SQPOLL   AA_MAY_CREATE
#define AA_MAY_OVERRIDE_CRED   AA_MAY_APPEND
#define AA_URING_PERM_MASK     (AA_MAY_OVERRIDE_CRED | AA_MAY_CREATE_SQPOLL)

#define PERMS_CHRS_MASK (MAY_READ | MAY_WRITE | AA_MAY_CREATE |                \
                         AA_MAY_DELETE | AA_MAY_LINK | AA_MAY_LOCK |        \
                         AA_MAY_EXEC | AA_EXEC_MMAP | AA_MAY_APPEND)

#define PERMS_NAMES_MASK (PERMS_CHRS_MASK | AA_MAY_OPEN | AA_MAY_RENAME |     \
                          AA_MAY_SETATTR | AA_MAY_GETATTR | AA_MAY_SETCRED | \
                          AA_MAY_GETCRED | AA_MAY_CHMOD | AA_MAY_CHOWN | \
                          AA_MAY_CHGRP | AA_MAY_MPROT | AA_MAY_SNAPSHOT | \
                          AA_MAY_STACK | AA_MAY_ONEXEC |                \
                          AA_MAY_CHANGE_PROFILE | AA_MAY_CHANGEHAT)

extern const char aa_file_perm_chrs[];
extern const char *aa_file_perm_names[];

struct aa_perms {
        u32 allow;
        u32 deny;        /* explicit deny, or conflict if allow also set */

        u32 subtree;        /* allow perm on full subtree only when allow is set */
        u32 cond;        /* set only when ~allow and ~deny */

        u32 kill;        /* set only when ~allow | deny */
        u32 complain;        /* accumulates only used when ~allow & ~deny */
        u32 prompt;        /* accumulates only used when ~allow & ~deny */

        u32 audit;        /* set only when allow is set */
        u32 quiet;        /* set only when ~allow | deny */
        u32 hide;        /* set only when  ~allow | deny */


        u32 xindex;
        u32 tag;        /* tag string index, if present */
        u32 label;        /* label string index, if present */
};

/*
 * Indexes are broken into a 24 bit index and 8 bit flag.
 * For the index to be valid there must be a value in the flag
 */
#define AA_INDEX_MASK                        0x00ffffff
#define AA_INDEX_FLAG_MASK                0xff000000
#define AA_INDEX_NONE                        0

#define ALL_PERMS_MASK 0xffffffff
extern struct aa_perms nullperms;
extern struct aa_perms allperms;

/**
 * aa_perms_accum_raw - accumulate perms with out masking off overlapping perms
 * @accum: perms struct to accumulate into
 * @addend: perms struct to add to @accum
 */
static inline void aa_perms_accum_raw(struct aa_perms *accum,
                                      struct aa_perms *addend)
{
        accum->deny |= addend->deny;
        accum->allow &= addend->allow & ~addend->deny;
        accum->audit |= addend->audit & addend->allow;
        accum->quiet &= addend->quiet & ~addend->allow;
        accum->kill |= addend->kill & ~addend->allow;
        accum->complain |= addend->complain & ~addend->allow & ~addend->deny;
        accum->cond |= addend->cond & ~addend->allow & ~addend->deny;
        accum->hide &= addend->hide & ~addend->allow;
        accum->prompt |= addend->prompt & ~addend->allow & ~addend->deny;
        accum->subtree |= addend->subtree & ~addend->deny;

        if (!accum->xindex)
                accum->xindex = addend->xindex;
        if (!accum->tag)
                accum->tag = addend->tag;
        if (!accum->label)
                accum->label = addend->label;
}

/**
 * aa_perms_accum - accumulate perms, masking off overlapping perms
 * @accum: perms struct to accumulate into
 * @addend: perms struct to add to @accum
 */
static inline void aa_perms_accum(struct aa_perms *accum,
                                  struct aa_perms *addend)
{
        accum->deny |= addend->deny;
        accum->allow &= addend->allow & ~accum->deny;
        accum->audit |= addend->audit & accum->allow;
        accum->quiet &= addend->quiet & ~accum->allow;
        accum->kill |= addend->kill & ~accum->allow;
        accum->complain |= addend->complain & ~accum->allow & ~accum->deny;
        accum->cond |= addend->cond & ~accum->allow & ~accum->deny;
        accum->hide &= addend->hide & ~accum->allow;
        accum->prompt |= addend->prompt & ~accum->allow & ~accum->deny;
        accum->subtree &= addend->subtree & ~accum->deny;

        if (!accum->xindex)
                accum->xindex = addend->xindex;
        if (!accum->tag)
                accum->tag = addend->tag;
        if (!accum->label)
                accum->label = addend->label;
}

#define xcheck(FN1, FN2)        \
({                                \
        int e, error = FN1;        \
        e = FN2;                \
        if (e)                        \
                error = e;        \
        error;                        \
})


/*
 * TODO: update for labels pointing to labels instead of profiles
 * TODO: optimize the walk, currently does subwalk of L2 for each P in L1
 * gah this doesn't allow for label compound check!!!!
 */
#define xcheck_ns_profile_profile(P1, P2, FN, args...)                \
({                                                                \
        int ____e = 0;                                                \
        if (P1->ns == P2->ns)                                        \
                ____e = FN((P1), (P2), args);                        \
        (____e);                                                \
})

#define xcheck_ns_profile_label(P, L, FN, args...)                \
({                                                                \
        struct aa_profile *__p2;                                \
        fn_for_each((L), __p2,                                        \
                    xcheck_ns_profile_profile((P), __p2, (FN), args));        \
})

#define xcheck_ns_labels(L1, L2, FN, args...)                        \
({                                                                \
        struct aa_profile *__p1;                                \
        fn_for_each((L1), __p1, FN(__p1, (L2), args));                \
})

/* Do the cross check but applying FN at the profiles level */
#define xcheck_labels_profiles(L1, L2, FN, args...)                \
        xcheck_ns_labels((L1), (L2), xcheck_ns_profile_label, (FN), args)

#define xcheck_labels(L1, L2, P, FN1, FN2)                        \
        xcheck(fn_for_each((L1), (P), (FN1)), fn_for_each((L2), (P), (FN2)))


extern struct aa_perms default_perms;


void aa_perm_mask_to_str(char *str, size_t str_size, const char *chrs,
                         u32 mask);
void aa_audit_perm_names(struct audit_buffer *ab, const char * const *names,
                         u32 mask);
void aa_audit_perm_mask(struct audit_buffer *ab, u32 mask, const char *chrs,
                        u32 chrsmask, const char * const *names, u32 namesmask);
void aa_apply_modes_to_perms(struct aa_profile *profile,
                             struct aa_perms *perms);
void aa_perms_accum(struct aa_perms *accum, struct aa_perms *addend);
void aa_perms_accum_raw(struct aa_perms *accum, struct aa_perms *addend);
void aa_profile_match_label(struct aa_profile *profile,
                            struct aa_ruleset *rules, struct aa_label *label,
                            int type, u32 request, struct aa_perms *perms);
int aa_check_perms(struct aa_profile *profile, struct aa_perms *perms,
                   u32 request, struct apparmor_audit_data *ad,
                   void (*cb)(struct audit_buffer *, void *));
#endif /* __AA_PERM_H */















































































































































































































































































































































































































































































































































































































    1 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
/*
 *
 * Author        Karsten Keil <kkeil@novell.com>
 *
 * Copyright 2008  by Karsten Keil <kkeil@novell.com>
 *
 * This code is free software; you can redistribute it and/or modify
 * it under the terms of the GNU LESSER GENERAL PUBLIC LICENSE
 * version 2.1 as published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU LESSER GENERAL PUBLIC LICENSE for more details.
 *
 */

#ifndef mISDNIF_H
#define mISDNIF_H

#include <linux/types.h>
#include <linux/errno.h>
#include <linux/socket.h>

/*
 * ABI Version 32 bit
 *
 * <8 bit> Major version
 *                - changed if any interface become backwards incompatible
 *
 * <8 bit> Minor version
 *              - changed if any interface is extended but backwards compatible
 *
 * <16 bit> Release number
 *              - should be incremented on every checkin
 */
#define        MISDN_MAJOR_VERSION        1
#define        MISDN_MINOR_VERSION        1
#define MISDN_RELEASE                29

/* primitives for information exchange
 * generell format
 * <16  bit  0 >
 * <8  bit command>
 *    BIT 8 = 1 LAYER private
 *    BIT 7 = 1 answer
 *    BIT 6 = 1 DATA
 * <8  bit target layer mask>
 *
 * Layer = 00 is reserved for general commands
   Layer = 01  L2 -> HW
   Layer = 02  HW -> L2
   Layer = 04  L3 -> L2
   Layer = 08  L2 -> L3
 * Layer = FF is reserved for broadcast commands
 */

#define MISDN_CMDMASK                0xff00
#define MISDN_LAYERMASK                0x00ff

/* generell commands */
#define OPEN_CHANNEL                0x0100
#define CLOSE_CHANNEL                0x0200
#define CONTROL_CHANNEL                0x0300
#define CHECK_DATA                0x0400

/* layer 2 -> layer 1 */
#define PH_ACTIVATE_REQ                0x0101
#define PH_DEACTIVATE_REQ        0x0201
#define PH_DATA_REQ                0x2001
#define MPH_ACTIVATE_REQ        0x0501
#define MPH_DEACTIVATE_REQ        0x0601
#define MPH_INFORMATION_REQ        0x0701
#define PH_CONTROL_REQ                0x0801

/* layer 1 -> layer 2 */
#define PH_ACTIVATE_IND                0x0102
#define PH_ACTIVATE_CNF                0x4102
#define PH_DEACTIVATE_IND        0x0202
#define PH_DEACTIVATE_CNF        0x4202
#define PH_DATA_IND                0x2002
#define PH_DATA_E_IND                0x3002
#define MPH_ACTIVATE_IND        0x0502
#define MPH_DEACTIVATE_IND        0x0602
#define MPH_INFORMATION_IND        0x0702
#define PH_DATA_CNF                0x6002
#define PH_CONTROL_IND                0x0802
#define PH_CONTROL_CNF                0x4802

/* layer 3 -> layer 2 */
#define DL_ESTABLISH_REQ        0x1004
#define DL_RELEASE_REQ                0x1104
#define DL_DATA_REQ                0x3004
#define DL_UNITDATA_REQ                0x3104
#define DL_INFORMATION_REQ        0x0004

/* layer 2 -> layer 3 */
#define DL_ESTABLISH_IND        0x1008
#define DL_ESTABLISH_CNF        0x5008
#define DL_RELEASE_IND                0x1108
#define DL_RELEASE_CNF                0x5108
#define DL_DATA_IND                0x3008
#define DL_UNITDATA_IND                0x3108
#define DL_INFORMATION_IND        0x0008

/* intern layer 2 management */
#define MDL_ASSIGN_REQ                0x1804
#define MDL_ASSIGN_IND                0x1904
#define MDL_REMOVE_REQ                0x1A04
#define MDL_REMOVE_IND                0x1B04
#define MDL_STATUS_UP_IND        0x1C04
#define MDL_STATUS_DOWN_IND        0x1D04
#define MDL_STATUS_UI_IND        0x1E04
#define MDL_ERROR_IND                0x1F04
#define MDL_ERROR_RSP                0x5F04

/* intern layer 2 */
#define DL_TIMER200_IND                0x7004
#define DL_TIMER203_IND                0x7304
#define DL_INTERN_MSG                0x7804

/* DL_INFORMATION_IND types */
#define DL_INFO_L2_CONNECT        0x0001
#define DL_INFO_L2_REMOVED        0x0002

/* PH_CONTROL types */
/* TOUCH TONE IS 0x20XX  XX "0"..."9", "A","B","C","D","*","#" */
#define DTMF_TONE_VAL                0x2000
#define DTMF_TONE_MASK                0x007F
#define DTMF_TONE_START                0x2100
#define DTMF_TONE_STOP                0x2200
#define DTMF_HFC_COEF                0x4000
#define DSP_CONF_JOIN                0x2403
#define DSP_CONF_SPLIT                0x2404
#define DSP_RECEIVE_OFF                0x2405
#define DSP_RECEIVE_ON                0x2406
#define DSP_ECHO_ON                0x2407
#define DSP_ECHO_OFF                0x2408
#define DSP_MIX_ON                0x2409
#define DSP_MIX_OFF                0x240a
#define DSP_DELAY                0x240b
#define DSP_JITTER                0x240c
#define DSP_TXDATA_ON                0x240d
#define DSP_TXDATA_OFF                0x240e
#define DSP_TX_DEJITTER                0x240f
#define DSP_TX_DEJ_OFF                0x2410
#define DSP_TONE_PATT_ON        0x2411
#define DSP_TONE_PATT_OFF        0x2412
#define DSP_VOL_CHANGE_TX        0x2413
#define DSP_VOL_CHANGE_RX        0x2414
#define DSP_BF_ENABLE_KEY        0x2415
#define DSP_BF_DISABLE                0x2416
#define DSP_BF_ACCEPT                0x2416
#define DSP_BF_REJECT                0x2417
#define DSP_PIPELINE_CFG        0x2418
#define HFC_VOL_CHANGE_TX        0x2601
#define HFC_VOL_CHANGE_RX        0x2602
#define HFC_SPL_LOOP_ON                0x2603
#define HFC_SPL_LOOP_OFF        0x2604
/* for T30 FAX and analog modem */
#define HW_MOD_FRM                0x4000
#define HW_MOD_FRH                0x4001
#define HW_MOD_FTM                0x4002
#define HW_MOD_FTH                0x4003
#define HW_MOD_FTS                0x4004
#define HW_MOD_CONNECT                0x4010
#define HW_MOD_OK                0x4011
#define HW_MOD_NOCARR                0x4012
#define HW_MOD_FCERROR                0x4013
#define HW_MOD_READY                0x4014
#define HW_MOD_LASTDATA                0x4015

/* DSP_TONE_PATT_ON parameter */
#define TONE_OFF                        0x0000
#define TONE_GERMAN_DIALTONE                0x0001
#define TONE_GERMAN_OLDDIALTONE                0x0002
#define TONE_AMERICAN_DIALTONE                0x0003
#define TONE_GERMAN_DIALPBX                0x0004
#define TONE_GERMAN_OLDDIALPBX                0x0005
#define TONE_AMERICAN_DIALPBX                0x0006
#define TONE_GERMAN_RINGING                0x0007
#define TONE_GERMAN_OLDRINGING                0x0008
#define TONE_AMERICAN_RINGPBX                0x000b
#define TONE_GERMAN_RINGPBX                0x000c
#define TONE_GERMAN_OLDRINGPBX                0x000d
#define TONE_AMERICAN_RINGING                0x000e
#define TONE_GERMAN_BUSY                0x000f
#define TONE_GERMAN_OLDBUSY                0x0010
#define TONE_AMERICAN_BUSY                0x0011
#define TONE_GERMAN_HANGUP                0x0012
#define TONE_GERMAN_OLDHANGUP                0x0013
#define TONE_AMERICAN_HANGUP                0x0014
#define TONE_SPECIAL_INFO                0x0015
#define TONE_GERMAN_GASSENBESETZT        0x0016
#define TONE_GERMAN_AUFSCHALTTON        0x0016

/* MPH_INFORMATION_IND */
#define L1_SIGNAL_LOS_OFF        0x0010
#define L1_SIGNAL_LOS_ON        0x0011
#define L1_SIGNAL_AIS_OFF        0x0012
#define L1_SIGNAL_AIS_ON        0x0013
#define L1_SIGNAL_RDI_OFF        0x0014
#define L1_SIGNAL_RDI_ON        0x0015
#define L1_SIGNAL_SLIP_RX        0x0020
#define L1_SIGNAL_SLIP_TX        0x0021

/*
 * protocol ids
 * D channel 1-31
 * B channel 33 - 63
 */

#define ISDN_P_NONE                0
#define ISDN_P_BASE                0
#define ISDN_P_TE_S0                0x01
#define ISDN_P_NT_S0                  0x02
#define ISDN_P_TE_E1                0x03
#define ISDN_P_NT_E1                  0x04
#define ISDN_P_TE_UP0                0x05
#define ISDN_P_NT_UP0                0x06

#define IS_ISDN_P_TE(p) ((p == ISDN_P_TE_S0) || (p == ISDN_P_TE_E1) || \
                                (p == ISDN_P_TE_UP0) || (p == ISDN_P_LAPD_TE))
#define IS_ISDN_P_NT(p) ((p == ISDN_P_NT_S0) || (p == ISDN_P_NT_E1) || \
                                (p == ISDN_P_NT_UP0) || (p == ISDN_P_LAPD_NT))
#define IS_ISDN_P_S0(p) ((p == ISDN_P_TE_S0) || (p == ISDN_P_NT_S0))
#define IS_ISDN_P_E1(p) ((p == ISDN_P_TE_E1) || (p == ISDN_P_NT_E1))
#define IS_ISDN_P_UP0(p) ((p == ISDN_P_TE_UP0) || (p == ISDN_P_NT_UP0))


#define ISDN_P_LAPD_TE                0x10
#define        ISDN_P_LAPD_NT                0x11

#define ISDN_P_B_MASK                0x1f
#define ISDN_P_B_START                0x20

#define ISDN_P_B_RAW                0x21
#define ISDN_P_B_HDLC                0x22
#define ISDN_P_B_X75SLP                0x23
#define ISDN_P_B_L2DTMF                0x24
#define ISDN_P_B_L2DSP                0x25
#define ISDN_P_B_L2DSPHDLC        0x26
#define ISDN_P_B_T30_FAX        0x27
#define ISDN_P_B_MODEM_ASYNC        0x28

#define OPTION_L2_PMX                1
#define OPTION_L2_PTP                2
#define OPTION_L2_FIXEDTEI        3
#define OPTION_L2_CLEANUP        4
#define OPTION_L1_HOLD                5

/* should be in sync with linux/kobject.h:KOBJ_NAME_LEN */
#define MISDN_MAX_IDLEN                20

struct mISDNhead {
        unsigned int        prim;
        unsigned int        id;
}  __packed;

#define MISDN_HEADER_LEN        sizeof(struct mISDNhead)
#define MAX_DATA_SIZE                2048
#define MAX_DATA_MEM                (MAX_DATA_SIZE + MISDN_HEADER_LEN)
#define MAX_DFRAME_LEN                260

#define MISDN_ID_ADDR_MASK        0xFFFF
#define MISDN_ID_TEI_MASK        0xFF00
#define MISDN_ID_SAPI_MASK        0x00FF
#define MISDN_ID_TEI_ANY        0x7F00

#define MISDN_ID_ANY                0xFFFF
#define MISDN_ID_NONE                0xFFFE

#define GROUP_TEI                127
#define TEI_SAPI                63
#define CTRL_SAPI                0

#define MISDN_MAX_CHANNEL        127
#define MISDN_CHMAP_SIZE        ((MISDN_MAX_CHANNEL + 1) >> 3)

#define SOL_MISDN        0

struct sockaddr_mISDN {
        sa_family_t    family;
        unsigned char        dev;
        unsigned char        channel;
        unsigned char        sapi;
        unsigned char        tei;
};

struct mISDNversion {
        unsigned char        major;
        unsigned char        minor;
        unsigned short        release;
};

struct mISDN_devinfo {
        u_int                        id;
        u_int                        Dprotocols;
        u_int                        Bprotocols;
        u_int                        protocol;
        u_char                        channelmap[MISDN_CHMAP_SIZE];
        u_int                        nrbchan;
        char                        name[MISDN_MAX_IDLEN];
};

struct mISDN_devrename {
        u_int                        id;
        char                        name[MISDN_MAX_IDLEN]; /* new name */
};

/* MPH_INFORMATION_REQ payload */
struct ph_info_ch {
        __u32 protocol;
        __u64 Flags;
};

struct ph_info_dch {
        struct ph_info_ch ch;
        __u16 state;
        __u16 num_bch;
};

struct ph_info {
        struct ph_info_dch dch;
        struct ph_info_ch  bch[];
};

/* timer device ioctl */
#define IMADDTIMER        _IOR('I', 64, int)
#define IMDELTIMER        _IOR('I', 65, int)

/* socket ioctls */
#define        IMGETVERSION        _IOR('I', 66, int)
#define        IMGETCOUNT        _IOR('I', 67, int)
#define IMGETDEVINFO        _IOR('I', 68, int)
#define IMCTRLREQ        _IOR('I', 69, int)
#define IMCLEAR_L2        _IOR('I', 70, int)
#define IMSETDEVNAME        _IOR('I', 71, struct mISDN_devrename)
#define IMHOLD_L1        _IOR('I', 72, int)

static inline int
test_channelmap(u_int nr, u_char *map)
{
        if (nr <= MISDN_MAX_CHANNEL)
                return map[nr >> 3] & (1 << (nr & 7));
        else
                return 0;
}

static inline void
set_channelmap(u_int nr, u_char *map)
{
        map[nr >> 3] |= (1 << (nr & 7));
}

static inline void
clear_channelmap(u_int nr, u_char *map)
{
        map[nr >> 3] &= ~(1 << (nr & 7));
}

/* CONTROL_CHANNEL parameters */
#define MISDN_CTRL_GETOP                0x0000
#define MISDN_CTRL_LOOP                        0x0001
#define MISDN_CTRL_CONNECT                0x0002
#define MISDN_CTRL_DISCONNECT                0x0004
#define MISDN_CTRL_RX_BUFFER                0x0008
#define MISDN_CTRL_PCMCONNECT                0x0010
#define MISDN_CTRL_PCMDISCONNECT        0x0020
#define MISDN_CTRL_SETPEER                0x0040
#define MISDN_CTRL_UNSETPEER                0x0080
#define MISDN_CTRL_RX_OFF                0x0100
#define MISDN_CTRL_FILL_EMPTY                0x0200
#define MISDN_CTRL_GETPEER                0x0400
#define MISDN_CTRL_L1_TIMER3                0x0800
#define MISDN_CTRL_HW_FEATURES_OP        0x2000
#define MISDN_CTRL_HW_FEATURES                0x2001
#define MISDN_CTRL_HFC_OP                0x4000
#define MISDN_CTRL_HFC_PCM_CONN                0x4001
#define MISDN_CTRL_HFC_PCM_DISC                0x4002
#define MISDN_CTRL_HFC_CONF_JOIN        0x4003
#define MISDN_CTRL_HFC_CONF_SPLIT        0x4004
#define MISDN_CTRL_HFC_RECEIVE_OFF        0x4005
#define MISDN_CTRL_HFC_RECEIVE_ON        0x4006
#define MISDN_CTRL_HFC_ECHOCAN_ON         0x4007
#define MISDN_CTRL_HFC_ECHOCAN_OFF         0x4008
#define MISDN_CTRL_HFC_WD_INIT                0x4009
#define MISDN_CTRL_HFC_WD_RESET                0x400A

/* special RX buffer value for MISDN_CTRL_RX_BUFFER request.p1 is the minimum
 * buffer size request.p2 the maximum. Using  MISDN_CTRL_RX_SIZE_IGNORE will
 * not change the value, but still read back the actual stetting.
 */
#define MISDN_CTRL_RX_SIZE_IGNORE        -1

/* socket options */
#define MISDN_TIME_STAMP                0x0001

struct mISDN_ctrl_req {
        int                op;
        int                channel;
        int                p1;
        int                p2;
};

/* muxer options */
#define MISDN_OPT_ALL                1
#define MISDN_OPT_TEIMGR        2

#ifdef __KERNEL__
#include <linux/list.h>
#include <linux/skbuff.h>
#include <linux/net.h>
#include <net/sock.h>
#include <linux/completion.h>

#define DEBUG_CORE                0x000000ff
#define DEBUG_CORE_FUNC                0x00000002
#define DEBUG_SOCKET                0x00000004
#define DEBUG_MANAGER                0x00000008
#define DEBUG_SEND_ERR                0x00000010
#define DEBUG_MSG_THREAD        0x00000020
#define DEBUG_QUEUE_FUNC        0x00000040
#define DEBUG_L1                0x0000ff00
#define DEBUG_L1_FSM                0x00000200
#define DEBUG_L2                0x00ff0000
#define DEBUG_L2_FSM                0x00020000
#define DEBUG_L2_CTRL                0x00040000
#define DEBUG_L2_RECV                0x00080000
#define DEBUG_L2_TEI                0x00100000
#define DEBUG_L2_TEIFSM                0x00200000
#define DEBUG_TIMER                0x01000000
#define DEBUG_CLOCK                0x02000000

#define mISDN_HEAD_P(s)                ((struct mISDNhead *)&s->cb[0])
#define mISDN_HEAD_PRIM(s)        (((struct mISDNhead *)&s->cb[0])->prim)
#define mISDN_HEAD_ID(s)        (((struct mISDNhead *)&s->cb[0])->id)

/* socket states */
#define MISDN_OPEN        1
#define MISDN_BOUND        2
#define MISDN_CLOSED        3

struct mISDNchannel;
struct mISDNdevice;
struct mISDNstack;
struct mISDNclock;

struct channel_req {
        u_int                        protocol;
        struct sockaddr_mISDN        adr;
        struct mISDNchannel        *ch;
};

typedef        int        (ctrl_func_t)(struct mISDNchannel *, u_int, void *);
typedef        int        (send_func_t)(struct mISDNchannel *, struct sk_buff *);
typedef int        (create_func_t)(struct channel_req *);

struct Bprotocol {
        struct list_head        list;
        char                        *name;
        u_int                        Bprotocols;
        create_func_t                *create;
};

struct mISDNchannel {
        struct list_head        list;
        u_int                        protocol;
        u_int                        nr;
        u_long                        opt;
        u_int                        addr;
        struct mISDNstack        *st;
        struct mISDNchannel        *peer;
        send_func_t                *send;
        send_func_t                *recv;
        ctrl_func_t                *ctrl;
};

struct mISDN_sock_list {
        struct hlist_head        head;
        rwlock_t                lock;
};

struct mISDN_sock {
        struct sock                sk;
        struct mISDNchannel        ch;
        u_int                        cmask;
        struct mISDNdevice        *dev;
};



struct mISDNdevice {
        struct mISDNchannel        D;
        u_int                        id;
        u_int                        Dprotocols;
        u_int                        Bprotocols;
        u_int                        nrbchan;
        u_char                        channelmap[MISDN_CHMAP_SIZE];
        struct list_head        bchannels;
        struct mISDNchannel        *teimgr;
        struct device                dev;
};

struct mISDNstack {
        u_long                        status;
        struct mISDNdevice        *dev;
        struct task_struct        *thread;
        struct completion        *notify;
        wait_queue_head_t        workq;
        struct sk_buff_head        msgq;
        struct list_head        layer2;
        struct mISDNchannel        *layer1;
        struct mISDNchannel        own;
        struct mutex                lmutex; /* protect lists */
        struct mISDN_sock_list        l1sock;
#ifdef MISDN_MSG_STATS
        u_int                        msg_cnt;
        u_int                        sleep_cnt;
        u_int                        stopped_cnt;
#endif
};

typedef        int        (clockctl_func_t)(void *, int);

struct        mISDNclock {
        struct list_head        list;
        char                        name[64];
        int                        pri;
        clockctl_func_t                *ctl;
        void                        *priv;
};

/* global alloc/queue functions */

static inline struct sk_buff *
mI_alloc_skb(unsigned int len, gfp_t gfp_mask)
{
        struct sk_buff        *skb;

        skb = alloc_skb(len + MISDN_HEADER_LEN, gfp_mask);
        if (likely(skb))
                skb_reserve(skb, MISDN_HEADER_LEN);
        return skb;
}

static inline struct sk_buff *
_alloc_mISDN_skb(u_int prim, u_int id, u_int len, void *dp, gfp_t gfp_mask)
{
        struct sk_buff        *skb = mI_alloc_skb(len, gfp_mask);
        struct mISDNhead *hh;

        if (!skb)
                return NULL;
        if (len)
                skb_put_data(skb, dp, len);
        hh = mISDN_HEAD_P(skb);
        hh->prim = prim;
        hh->id = id;
        return skb;
}

static inline void
_queue_data(struct mISDNchannel *ch, u_int prim,
    u_int id, u_int len, void *dp, gfp_t gfp_mask)
{
        struct sk_buff                *skb;

        if (!ch->peer)
                return;
        skb = _alloc_mISDN_skb(prim, id, len, dp, gfp_mask);
        if (!skb)
                return;
        if (ch->recv(ch->peer, skb))
                dev_kfree_skb(skb);
}

/* global register/unregister functions */

extern int        mISDN_register_device(struct mISDNdevice *,
                                        struct device *parent, char *name);
extern void        mISDN_unregister_device(struct mISDNdevice *);
extern int        mISDN_register_Bprotocol(struct Bprotocol *);
extern void        mISDN_unregister_Bprotocol(struct Bprotocol *);
extern struct mISDNclock *mISDN_register_clock(char *, int, clockctl_func_t *,
                                                void *);
extern void        mISDN_unregister_clock(struct mISDNclock *);

static inline struct mISDNdevice *dev_to_mISDN(const struct device *dev)
{
        if (dev)
                return dev_get_drvdata(dev);
        else
                return NULL;
}

extern void        set_channel_address(struct mISDNchannel *, u_int, u_int);
extern void        mISDN_clock_update(struct mISDNclock *, int, ktime_t *);
extern unsigned short mISDN_clock_get(void);
extern const char *mISDNDevName4ch(struct mISDNchannel *);

#endif /* __KERNEL__ */
#endif /* mISDNIF_H */









































































































































































































































































































































































































































































































































































































































    8 
    8 




    8 



    8 

    8 

    8 


    8 


















    8 




    8 










    8 

    8 
    8 




























































    8 





















    8 

    8 

    8 






    8 


    8 



































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2017 Facebook
 */

#include <linux/kernel.h>
#include <linux/blkdev.h>
#include <linux/build_bug.h>
#include <linux/debugfs.h>

#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-debugfs.h"
#include "blk-mq-sched.h"
#include "blk-rq-qos.h"

static int queue_poll_stat_show(void *data, struct seq_file *m)
{
        return 0;
}

static void *queue_requeue_list_start(struct seq_file *m, loff_t *pos)
        __acquires(&q->requeue_lock)
{
        struct request_queue *q = m->private;

        spin_lock_irq(&q->requeue_lock);
        return seq_list_start(&q->requeue_list, *pos);
}

static void *queue_requeue_list_next(struct seq_file *m, void *v, loff_t *pos)
{
        struct request_queue *q = m->private;

        return seq_list_next(v, &q->requeue_list, pos);
}

static void queue_requeue_list_stop(struct seq_file *m, void *v)
        __releases(&q->requeue_lock)
{
        struct request_queue *q = m->private;

        spin_unlock_irq(&q->requeue_lock);
}

static const struct seq_operations queue_requeue_list_seq_ops = {
        .start        = queue_requeue_list_start,
        .next        = queue_requeue_list_next,
        .stop        = queue_requeue_list_stop,
        .show        = blk_mq_debugfs_rq_show,
};

static int blk_flags_show(struct seq_file *m, const unsigned long flags,
                          const char *const *flag_name, int flag_name_count)
{
        bool sep = false;
        int i;

        for (i = 0; i < sizeof(flags) * BITS_PER_BYTE; i++) {
                if (!(flags & BIT(i)))
                        continue;
                if (sep)
                        seq_puts(m, "|");
                sep = true;
                if (i < flag_name_count && flag_name[i])
                        seq_puts(m, flag_name[i]);
                else
                        seq_printf(m, "%d", i);
        }
        return 0;
}

static int queue_pm_only_show(void *data, struct seq_file *m)
{
        struct request_queue *q = data;

        seq_printf(m, "%d\n", atomic_read(&q->pm_only));
        return 0;
}

#define QUEUE_FLAG_NAME(name) [QUEUE_FLAG_##name] = #name
static const char *const blk_queue_flag_name[] = {
        QUEUE_FLAG_NAME(DYING),
        QUEUE_FLAG_NAME(NOMERGES),
        QUEUE_FLAG_NAME(SAME_COMP),
        QUEUE_FLAG_NAME(FAIL_IO),
        QUEUE_FLAG_NAME(NOXMERGES),
        QUEUE_FLAG_NAME(SAME_FORCE),
        QUEUE_FLAG_NAME(INIT_DONE),
        QUEUE_FLAG_NAME(STATS),
        QUEUE_FLAG_NAME(REGISTERED),
        QUEUE_FLAG_NAME(QUIESCED),
        QUEUE_FLAG_NAME(RQ_ALLOC_TIME),
        QUEUE_FLAG_NAME(HCTX_ACTIVE),
        QUEUE_FLAG_NAME(SQ_SCHED),
        QUEUE_FLAG_NAME(DISABLE_WBT_DEF),
        QUEUE_FLAG_NAME(NO_ELV_SWITCH),
        QUEUE_FLAG_NAME(QOS_ENABLED),
        QUEUE_FLAG_NAME(BIO_ISSUE_TIME),
};
#undef QUEUE_FLAG_NAME

static int queue_state_show(void *data, struct seq_file *m)
{
        struct request_queue *q = data;

        BUILD_BUG_ON(ARRAY_SIZE(blk_queue_flag_name) != QUEUE_FLAG_MAX);
        blk_flags_show(m, q->queue_flags, blk_queue_flag_name,
                       ARRAY_SIZE(blk_queue_flag_name));
        seq_puts(m, "\n");
        return 0;
}

static ssize_t queue_state_write(void *data, const char __user *buf,
                                 size_t count, loff_t *ppos)
{
        struct request_queue *q = data;
        char opbuf[16] = { }, *op;

        /*
         * The "state" attribute is removed when the queue is removed.  Don't
         * allow setting the state on a dying queue to avoid a use-after-free.
         */
        if (blk_queue_dying(q))
                return -ENOENT;

        if (count >= sizeof(opbuf)) {
                pr_err("%s: operation too long\n", __func__);
                goto inval;
        }

        if (copy_from_user(opbuf, buf, count))
                return -EFAULT;
        op = strstrip(opbuf);
        if (strcmp(op, "run") == 0) {
                blk_mq_run_hw_queues(q, true);
        } else if (strcmp(op, "start") == 0) {
                blk_mq_start_stopped_hw_queues(q, true);
        } else if (strcmp(op, "kick") == 0) {
                blk_mq_kick_requeue_list(q);
        } else {
                pr_err("%s: unsupported operation '%s'\n", __func__, op);
inval:
                pr_err("%s: use 'run', 'start' or 'kick'\n", __func__);
                return -EINVAL;
        }
        return count;
}

static const struct blk_mq_debugfs_attr blk_mq_debugfs_queue_attrs[] = {
        { "poll_stat", 0400, queue_poll_stat_show },
        { "requeue_list", 0400, .seq_ops = &queue_requeue_list_seq_ops },
        { "pm_only", 0600, queue_pm_only_show, NULL },
        { "state", 0600, queue_state_show, queue_state_write },
        { "zone_wplugs", 0400, queue_zone_wplugs_show, NULL },
        { },
};

#define HCTX_STATE_NAME(name) [BLK_MQ_S_##name] = #name
static const char *const hctx_state_name[] = {
        HCTX_STATE_NAME(STOPPED),
        HCTX_STATE_NAME(TAG_ACTIVE),
        HCTX_STATE_NAME(SCHED_RESTART),
        HCTX_STATE_NAME(INACTIVE),
};
#undef HCTX_STATE_NAME

static int hctx_state_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;

        BUILD_BUG_ON(ARRAY_SIZE(hctx_state_name) != BLK_MQ_S_MAX);
        blk_flags_show(m, hctx->state, hctx_state_name,
                       ARRAY_SIZE(hctx_state_name));
        seq_puts(m, "\n");
        return 0;
}

#define HCTX_FLAG_NAME(name) [ilog2(BLK_MQ_F_##name)] = #name
static const char *const hctx_flag_name[] = {
        HCTX_FLAG_NAME(TAG_QUEUE_SHARED),
        HCTX_FLAG_NAME(STACKING),
        HCTX_FLAG_NAME(TAG_HCTX_SHARED),
        HCTX_FLAG_NAME(BLOCKING),
        HCTX_FLAG_NAME(TAG_RR),
        HCTX_FLAG_NAME(NO_SCHED_BY_DEFAULT),
};
#undef HCTX_FLAG_NAME

static int hctx_flags_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;

        BUILD_BUG_ON(ARRAY_SIZE(hctx_flag_name) != ilog2(BLK_MQ_F_MAX));

        blk_flags_show(m, hctx->flags, hctx_flag_name,
                        ARRAY_SIZE(hctx_flag_name));
        seq_puts(m, "\n");
        return 0;
}

#define CMD_FLAG_NAME(name) [__REQ_##name] = #name
static const char *const cmd_flag_name[] = {
        CMD_FLAG_NAME(FAILFAST_DEV),
        CMD_FLAG_NAME(FAILFAST_TRANSPORT),
        CMD_FLAG_NAME(FAILFAST_DRIVER),
        CMD_FLAG_NAME(SYNC),
        CMD_FLAG_NAME(META),
        CMD_FLAG_NAME(PRIO),
        CMD_FLAG_NAME(NOMERGE),
        CMD_FLAG_NAME(IDLE),
        CMD_FLAG_NAME(INTEGRITY),
        CMD_FLAG_NAME(FUA),
        CMD_FLAG_NAME(PREFLUSH),
        CMD_FLAG_NAME(RAHEAD),
        CMD_FLAG_NAME(BACKGROUND),
        CMD_FLAG_NAME(NOWAIT),
        CMD_FLAG_NAME(POLLED),
        CMD_FLAG_NAME(ALLOC_CACHE),
        CMD_FLAG_NAME(SWAP),
        CMD_FLAG_NAME(DRV),
        CMD_FLAG_NAME(FS_PRIVATE),
        CMD_FLAG_NAME(ATOMIC),
        CMD_FLAG_NAME(NOUNMAP),
};
#undef CMD_FLAG_NAME

#define RQF_NAME(name) [__RQF_##name] = #name
static const char *const rqf_name[] = {
        RQF_NAME(STARTED),
        RQF_NAME(FLUSH_SEQ),
        RQF_NAME(MIXED_MERGE),
        RQF_NAME(DONTPREP),
        RQF_NAME(SCHED_TAGS),
        RQF_NAME(USE_SCHED),
        RQF_NAME(FAILED),
        RQF_NAME(QUIET),
        RQF_NAME(IO_STAT),
        RQF_NAME(PM),
        RQF_NAME(HASHED),
        RQF_NAME(STATS),
        RQF_NAME(SPECIAL_PAYLOAD),
        RQF_NAME(ZONE_WRITE_PLUGGING),
        RQF_NAME(TIMED_OUT),
        RQF_NAME(RESV),
};
#undef RQF_NAME

static const char *const blk_mq_rq_state_name_array[] = {
        [MQ_RQ_IDLE]                = "idle",
        [MQ_RQ_IN_FLIGHT]        = "in_flight",
        [MQ_RQ_COMPLETE]        = "complete",
};

static const char *blk_mq_rq_state_name(enum mq_rq_state rq_state)
{
        if (WARN_ON_ONCE((unsigned int)rq_state >=
                         ARRAY_SIZE(blk_mq_rq_state_name_array)))
                return "(?)";
        return blk_mq_rq_state_name_array[rq_state];
}

int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq)
{
        const struct blk_mq_ops *const mq_ops = rq->q->mq_ops;
        const enum req_op op = req_op(rq);
        const char *op_str = blk_op_str(op);

        BUILD_BUG_ON(ARRAY_SIZE(cmd_flag_name) != __REQ_NR_BITS);
        BUILD_BUG_ON(ARRAY_SIZE(rqf_name) != __RQF_BITS);

        seq_printf(m, "%p {.op=", rq);
        if (strcmp(op_str, "UNKNOWN") == 0)
                seq_printf(m, "%u", op);
        else
                seq_printf(m, "%s", op_str);
        seq_puts(m, ", .cmd_flags=");
        blk_flags_show(m, (__force unsigned int)(rq->cmd_flags & ~REQ_OP_MASK),
                       cmd_flag_name, ARRAY_SIZE(cmd_flag_name));
        seq_puts(m, ", .rq_flags=");
        blk_flags_show(m, (__force unsigned int)rq->rq_flags, rqf_name,
                       ARRAY_SIZE(rqf_name));
        seq_printf(m, ", .state=%s", blk_mq_rq_state_name(blk_mq_rq_state(rq)));
        seq_printf(m, ", .tag=%d, .internal_tag=%d", rq->tag,
                   rq->internal_tag);
        if (mq_ops->show_rq)
                mq_ops->show_rq(m, rq);
        seq_puts(m, "}\n");
        return 0;
}
EXPORT_SYMBOL_GPL(__blk_mq_debugfs_rq_show);

int blk_mq_debugfs_rq_show(struct seq_file *m, void *v)
{
        return __blk_mq_debugfs_rq_show(m, list_entry_rq(v));
}
EXPORT_SYMBOL_GPL(blk_mq_debugfs_rq_show);

static void *hctx_dispatch_start(struct seq_file *m, loff_t *pos)
        __acquires(&hctx->lock)
{
        struct blk_mq_hw_ctx *hctx = m->private;

        spin_lock(&hctx->lock);
        return seq_list_start(&hctx->dispatch, *pos);
}

static void *hctx_dispatch_next(struct seq_file *m, void *v, loff_t *pos)
{
        struct blk_mq_hw_ctx *hctx = m->private;

        return seq_list_next(v, &hctx->dispatch, pos);
}

static void hctx_dispatch_stop(struct seq_file *m, void *v)
        __releases(&hctx->lock)
{
        struct blk_mq_hw_ctx *hctx = m->private;

        spin_unlock(&hctx->lock);
}

static const struct seq_operations hctx_dispatch_seq_ops = {
        .start        = hctx_dispatch_start,
        .next        = hctx_dispatch_next,
        .stop        = hctx_dispatch_stop,
        .show        = blk_mq_debugfs_rq_show,
};

struct show_busy_params {
        struct seq_file                *m;
        struct blk_mq_hw_ctx        *hctx;
};

/*
 * Note: the state of a request may change while this function is in progress,
 * e.g. due to a concurrent blk_mq_finish_request() call. Returns true to
 * keep iterating requests.
 */
static bool hctx_show_busy_rq(struct request *rq, void *data)
{
        const struct show_busy_params *params = data;

        if (rq->mq_hctx == params->hctx)
                __blk_mq_debugfs_rq_show(params->m, rq);

        return true;
}

static int hctx_busy_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;
        struct show_busy_params params = { .m = m, .hctx = hctx };
        int res;

        res = mutex_lock_interruptible(&hctx->queue->elevator_lock);
        if (res)
                return res;
        blk_mq_tagset_busy_iter(hctx->queue->tag_set, hctx_show_busy_rq,
                                &params);
        mutex_unlock(&hctx->queue->elevator_lock);

        return 0;
}

static const char *const hctx_types[] = {
        [HCTX_TYPE_DEFAULT]        = "default",
        [HCTX_TYPE_READ]        = "read",
        [HCTX_TYPE_POLL]        = "poll",
};

static int hctx_type_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;

        BUILD_BUG_ON(ARRAY_SIZE(hctx_types) != HCTX_MAX_TYPES);
        seq_printf(m, "%s\n", hctx_types[hctx->type]);
        return 0;
}

static int hctx_ctx_map_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;

        sbitmap_bitmap_show(&hctx->ctx_map, m);
        return 0;
}

static void blk_mq_debugfs_tags_show(struct seq_file *m,
                                     struct blk_mq_tags *tags)
{
        seq_printf(m, "nr_tags=%u\n", tags->nr_tags);
        seq_printf(m, "nr_reserved_tags=%u\n", tags->nr_reserved_tags);
        seq_printf(m, "active_queues=%d\n",
                   READ_ONCE(tags->active_queues));

        seq_puts(m, "\nbitmap_tags:\n");
        sbitmap_queue_show(&tags->bitmap_tags, m);

        if (tags->nr_reserved_tags) {
                seq_puts(m, "\nbreserved_tags:\n");
                sbitmap_queue_show(&tags->breserved_tags, m);
        }
}

static int hctx_tags_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;
        struct request_queue *q = hctx->queue;
        int res;

        res = mutex_lock_interruptible(&q->elevator_lock);
        if (res)
                return res;
        if (hctx->tags)
                blk_mq_debugfs_tags_show(m, hctx->tags);
        mutex_unlock(&q->elevator_lock);

        return 0;
}

static int hctx_tags_bitmap_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;
        struct request_queue *q = hctx->queue;
        int res;

        res = mutex_lock_interruptible(&q->elevator_lock);
        if (res)
                return res;
        if (hctx->tags)
                sbitmap_bitmap_show(&hctx->tags->bitmap_tags.sb, m);
        mutex_unlock(&q->elevator_lock);

        return 0;
}

static int hctx_sched_tags_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;
        struct request_queue *q = hctx->queue;
        int res;

        res = mutex_lock_interruptible(&q->elevator_lock);
        if (res)
                return res;
        if (hctx->sched_tags)
                blk_mq_debugfs_tags_show(m, hctx->sched_tags);
        mutex_unlock(&q->elevator_lock);

        return 0;
}

static int hctx_sched_tags_bitmap_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;
        struct request_queue *q = hctx->queue;
        int res;

        res = mutex_lock_interruptible(&q->elevator_lock);
        if (res)
                return res;
        if (hctx->sched_tags)
                sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags.sb, m);
        mutex_unlock(&q->elevator_lock);

        return 0;
}

static int hctx_active_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;

        seq_printf(m, "%d\n", __blk_mq_active_requests(hctx));
        return 0;
}

static int hctx_dispatch_busy_show(void *data, struct seq_file *m)
{
        struct blk_mq_hw_ctx *hctx = data;

        seq_printf(m, "%u\n", hctx->dispatch_busy);
        return 0;
}

#define CTX_RQ_SEQ_OPS(name, type)                                        \
static void *ctx_##name##_rq_list_start(struct seq_file *m, loff_t *pos) \
        __acquires(&ctx->lock)                                                \
{                                                                        \
        struct blk_mq_ctx *ctx = m->private;                                \
                                                                        \
        spin_lock(&ctx->lock);                                                \
        return seq_list_start(&ctx->rq_lists[type], *pos);                \
}                                                                        \
                                                                        \
static void *ctx_##name##_rq_list_next(struct seq_file *m, void *v,        \
                                     loff_t *pos)                        \
{                                                                        \
        struct blk_mq_ctx *ctx = m->private;                                \
                                                                        \
        return seq_list_next(v, &ctx->rq_lists[type], pos);                \
}                                                                        \
                                                                        \
static void ctx_##name##_rq_list_stop(struct seq_file *m, void *v)        \
        __releases(&ctx->lock)                                                \
{                                                                        \
        struct blk_mq_ctx *ctx = m->private;                                \
                                                                        \
        spin_unlock(&ctx->lock);                                        \
}                                                                        \
                                                                        \
static const struct seq_operations ctx_##name##_rq_list_seq_ops = {        \
        .start        = ctx_##name##_rq_list_start,                                \
        .next        = ctx_##name##_rq_list_next,                                \
        .stop        = ctx_##name##_rq_list_stop,                                \
        .show        = blk_mq_debugfs_rq_show,                                \
}

CTX_RQ_SEQ_OPS(default, HCTX_TYPE_DEFAULT);
CTX_RQ_SEQ_OPS(read, HCTX_TYPE_READ);
CTX_RQ_SEQ_OPS(poll, HCTX_TYPE_POLL);

static int blk_mq_debugfs_show(struct seq_file *m, void *v)
{
        const struct blk_mq_debugfs_attr *attr = m->private;
        void *data = debugfs_get_aux(m->file);

        return attr->show(data, m);
}

static ssize_t blk_mq_debugfs_write(struct file *file, const char __user *buf,
                                    size_t count, loff_t *ppos)
{
        struct seq_file *m = file->private_data;
        const struct blk_mq_debugfs_attr *attr = m->private;
        void *data = debugfs_get_aux(file);

        /*
         * Attributes that only implement .seq_ops are read-only and 'attr' is
         * the same with 'data' in this case.
         */
        if (attr == data || !attr->write)
                return -EPERM;

        return attr->write(data, buf, count, ppos);
}

static int blk_mq_debugfs_open(struct inode *inode, struct file *file)
{
        const struct blk_mq_debugfs_attr *attr = inode->i_private;
        void *data = debugfs_get_aux(file);
        struct seq_file *m;
        int ret;

        if (attr->seq_ops) {
                ret = seq_open(file, attr->seq_ops);
                if (!ret) {
                        m = file->private_data;
                        m->private = data;
                }
                return ret;
        }

        if (WARN_ON_ONCE(!attr->show))
                return -EPERM;

        return single_open(file, blk_mq_debugfs_show, inode->i_private);
}

static int blk_mq_debugfs_release(struct inode *inode, struct file *file)
{
        const struct blk_mq_debugfs_attr *attr = inode->i_private;

        if (attr->show)
                return single_release(inode, file);

        return seq_release(inode, file);
}

static const struct file_operations blk_mq_debugfs_fops = {
        .open                = blk_mq_debugfs_open,
        .read                = seq_read,
        .write                = blk_mq_debugfs_write,
        .llseek                = seq_lseek,
        .release        = blk_mq_debugfs_release,
};

static const struct blk_mq_debugfs_attr blk_mq_debugfs_hctx_attrs[] = {
        {"state", 0400, hctx_state_show},
        {"flags", 0400, hctx_flags_show},
        {"dispatch", 0400, .seq_ops = &hctx_dispatch_seq_ops},
        {"busy", 0400, hctx_busy_show},
        {"ctx_map", 0400, hctx_ctx_map_show},
        {"tags", 0400, hctx_tags_show},
        {"tags_bitmap", 0400, hctx_tags_bitmap_show},
        {"sched_tags", 0400, hctx_sched_tags_show},
        {"sched_tags_bitmap", 0400, hctx_sched_tags_bitmap_show},
        {"active", 0400, hctx_active_show},
        {"dispatch_busy", 0400, hctx_dispatch_busy_show},
        {"type", 0400, hctx_type_show},
        {},
};

static const struct blk_mq_debugfs_attr blk_mq_debugfs_ctx_attrs[] = {
        {"default_rq_list", 0400, .seq_ops = &ctx_default_rq_list_seq_ops},
        {"read_rq_list", 0400, .seq_ops = &ctx_read_rq_list_seq_ops},
        {"poll_rq_list", 0400, .seq_ops = &ctx_poll_rq_list_seq_ops},
        {},
};

static void debugfs_create_files(struct dentry *parent, void *data,
                                 const struct blk_mq_debugfs_attr *attr)
{
        if (IS_ERR_OR_NULL(parent))
                return;

        for (; attr->name; attr++)
                debugfs_create_file_aux(attr->name, attr->mode, parent,
                                    (void *)attr, data, &blk_mq_debugfs_fops);
}

void blk_mq_debugfs_register(struct request_queue *q)
{
        struct blk_mq_hw_ctx *hctx;
        unsigned long i;

        debugfs_create_files(q->debugfs_dir, q, blk_mq_debugfs_queue_attrs);

        queue_for_each_hw_ctx(q, hctx, i) {
                if (!hctx->debugfs_dir)
                        blk_mq_debugfs_register_hctx(q, hctx);
        }

        if (q->rq_qos) {
                struct rq_qos *rqos = q->rq_qos;

                while (rqos) {
                        blk_mq_debugfs_register_rqos(rqos);
                        rqos = rqos->next;
                }
        }
}

static void blk_mq_debugfs_register_ctx(struct blk_mq_hw_ctx *hctx,
                                        struct blk_mq_ctx *ctx)
{
        struct dentry *ctx_dir;
        char name[20];

        snprintf(name, sizeof(name), "cpu%u", ctx->cpu);
        ctx_dir = debugfs_create_dir(name, hctx->debugfs_dir);

        debugfs_create_files(ctx_dir, ctx, blk_mq_debugfs_ctx_attrs);
}

void blk_mq_debugfs_register_hctx(struct request_queue *q,
                                  struct blk_mq_hw_ctx *hctx)
{
        struct blk_mq_ctx *ctx;
        char name[20];
        int i;

        if (!q->debugfs_dir)
                return;

        snprintf(name, sizeof(name), "hctx%u", hctx->queue_num);
        hctx->debugfs_dir = debugfs_create_dir(name, q->debugfs_dir);

        debugfs_create_files(hctx->debugfs_dir, hctx, blk_mq_debugfs_hctx_attrs);

        hctx_for_each_ctx(hctx, ctx, i)
                blk_mq_debugfs_register_ctx(hctx, ctx);
}

void blk_mq_debugfs_unregister_hctx(struct blk_mq_hw_ctx *hctx)
{
        if (!hctx->queue->debugfs_dir)
                return;
        debugfs_remove_recursive(hctx->debugfs_dir);
        hctx->sched_debugfs_dir = NULL;
        hctx->debugfs_dir = NULL;
}

void blk_mq_debugfs_register_hctxs(struct request_queue *q)
{
        struct blk_mq_hw_ctx *hctx;
        unsigned long i;

        queue_for_each_hw_ctx(q, hctx, i)
                blk_mq_debugfs_register_hctx(q, hctx);
}

void blk_mq_debugfs_unregister_hctxs(struct request_queue *q)
{
        struct blk_mq_hw_ctx *hctx;
        unsigned long i;

        queue_for_each_hw_ctx(q, hctx, i)
                blk_mq_debugfs_unregister_hctx(hctx);
}

void blk_mq_debugfs_register_sched(struct request_queue *q)
{
        struct elevator_type *e = q->elevator->type;

        lockdep_assert_held(&q->debugfs_mutex);

        /*
         * If the parent directory has not been created yet, return, we will be
         * called again later on and the directory/files will be created then.
         */
        if (!q->debugfs_dir)
                return;

        if (!e->queue_debugfs_attrs)
                return;

        q->sched_debugfs_dir = debugfs_create_dir("sched", q->debugfs_dir);

        debugfs_create_files(q->sched_debugfs_dir, q, e->queue_debugfs_attrs);
}

void blk_mq_debugfs_unregister_sched(struct request_queue *q)
{
        lockdep_assert_held(&q->debugfs_mutex);

        debugfs_remove_recursive(q->sched_debugfs_dir);
        q->sched_debugfs_dir = NULL;
}

static const char *rq_qos_id_to_name(enum rq_qos_id id)
{
        switch (id) {
        case RQ_QOS_WBT:
                return "wbt";
        case RQ_QOS_LATENCY:
                return "latency";
        case RQ_QOS_COST:
                return "cost";
        }
        return "unknown";
}

void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos)
{
        lockdep_assert_held(&rqos->disk->queue->debugfs_mutex);

        if (!rqos->disk->queue->debugfs_dir)
                return;
        debugfs_remove_recursive(rqos->debugfs_dir);
        rqos->debugfs_dir = NULL;
}

void blk_mq_debugfs_register_rqos(struct rq_qos *rqos)
{
        struct request_queue *q = rqos->disk->queue;
        const char *dir_name = rq_qos_id_to_name(rqos->id);

        lockdep_assert_held(&q->debugfs_mutex);

        if (rqos->debugfs_dir || !rqos->ops->debugfs_attrs)
                return;

        if (!q->rqos_debugfs_dir)
                q->rqos_debugfs_dir = debugfs_create_dir("rqos",
                                                         q->debugfs_dir);

        rqos->debugfs_dir = debugfs_create_dir(dir_name, q->rqos_debugfs_dir);
        debugfs_create_files(rqos->debugfs_dir, rqos, rqos->ops->debugfs_attrs);
}

void blk_mq_debugfs_register_sched_hctx(struct request_queue *q,
                                        struct blk_mq_hw_ctx *hctx)
{
        struct elevator_type *e = q->elevator->type;

        lockdep_assert_held(&q->debugfs_mutex);

        /*
         * If the parent debugfs directory has not been created yet, return;
         * We will be called again later on with appropriate parent debugfs
         * directory from blk_register_queue()
         */
        if (!hctx->debugfs_dir)
                return;

        if (!e->hctx_debugfs_attrs)
                return;

        hctx->sched_debugfs_dir = debugfs_create_dir("sched",
                                                     hctx->debugfs_dir);
        debugfs_create_files(hctx->sched_debugfs_dir, hctx,
                             e->hctx_debugfs_attrs);
}

void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx)
{
        lockdep_assert_held(&hctx->queue->debugfs_mutex);

        if (!hctx->queue->debugfs_dir)
                return;
        debugfs_remove_recursive(hctx->sched_debugfs_dir);
        hctx->sched_debugfs_dir = NULL;
}






















































































































































































































































    1 




    1 






















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
// SPDX-License-Identifier: GPL-2.0-only
/*
 * IPV6 GSO/GRO offload support
 * Linux INET implementation
 *
 * Copyright (C) 2016 secunet Security Networks AG
 * Author: Steffen Klassert <steffen.klassert@secunet.com>
 *
 * ESP GRO support
 */

#include <linux/skbuff.h>
#include <linux/init.h>
#include <net/protocol.h>
#include <crypto/aead.h>
#include <crypto/authenc.h>
#include <linux/err.h>
#include <linux/module.h>
#include <net/gro.h>
#include <net/gso.h>
#include <net/ip.h>
#include <net/xfrm.h>
#include <net/esp.h>
#include <linux/scatterlist.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <net/ip6_route.h>
#include <net/ipv6.h>
#include <linux/icmpv6.h>

static __u16 esp6_nexthdr_esp_offset(struct ipv6hdr *ipv6_hdr, int nhlen)
{
        int off = sizeof(struct ipv6hdr);
        struct ipv6_opt_hdr *exthdr;

        /* ESP or ESPINUDP */
        if (likely(ipv6_hdr->nexthdr == NEXTHDR_ESP ||
                   ipv6_hdr->nexthdr == NEXTHDR_UDP))
                return offsetof(struct ipv6hdr, nexthdr);

        while (off < nhlen) {
                exthdr = (void *)ipv6_hdr + off;
                if (exthdr->nexthdr == NEXTHDR_ESP)
                        return off;

                off += ipv6_optlen(exthdr);
        }

        return 0;
}

static struct sk_buff *esp6_gro_receive(struct list_head *head,
                                        struct sk_buff *skb)
{
        int offset = skb_gro_offset(skb);
        struct xfrm_offload *xo;
        struct xfrm_state *x;
        int encap_type = 0;
        __be32 seq;
        __be32 spi;
        int nhoff;

        if (NAPI_GRO_CB(skb)->proto == IPPROTO_UDP)
                encap_type = UDP_ENCAP_ESPINUDP;

        if (!pskb_pull(skb, offset))
                return NULL;

        if (xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq) != 0)
                goto out;

        xo = xfrm_offload(skb);
        if (!xo || !(xo->flags & CRYPTO_DONE)) {
                struct sec_path *sp = secpath_set(skb);

                if (!sp)
                        goto out;

                if (sp->len == XFRM_MAX_DEPTH)
                        goto out_reset;

                x = xfrm_input_state_lookup(dev_net(skb->dev), skb->mark,
                                            (xfrm_address_t *)&ipv6_hdr(skb)->daddr,
                                            spi, IPPROTO_ESP, AF_INET6);

                if (unlikely(x && x->dir && x->dir != XFRM_SA_DIR_IN)) {
                        /* non-offload path will record the error and audit log */
                        xfrm_state_put(x);
                        x = NULL;
                }

                if (!x)
                        goto out_reset;

                skb->mark = xfrm_smark_get(skb->mark, x);

                sp->xvec[sp->len++] = x;
                sp->olen++;

                xo = xfrm_offload(skb);
                if (!xo)
                        goto out_reset;
        }

        xo->flags |= XFRM_GRO;

        nhoff = esp6_nexthdr_esp_offset(ipv6_hdr(skb), offset);
        if (!nhoff)
                goto out;

        IP6CB(skb)->nhoff = nhoff;
        XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6 = NULL;
        XFRM_SPI_SKB_CB(skb)->family = AF_INET6;
        XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct ipv6hdr, daddr);
        XFRM_SPI_SKB_CB(skb)->seq = seq;

        /* We don't need to handle errors from xfrm_input, it does all
         * the error handling and frees the resources on error. */
        xfrm_input(skb, IPPROTO_ESP, spi, encap_type);

        return ERR_PTR(-EINPROGRESS);
out_reset:
        secpath_reset(skb);
out:
        skb_push(skb, offset);
        NAPI_GRO_CB(skb)->same_flow = 0;
        NAPI_GRO_CB(skb)->flush = 1;

        return NULL;
}

static void esp6_gso_encap(struct xfrm_state *x, struct sk_buff *skb)
{
        struct ip_esp_hdr *esph;
        struct ipv6hdr *iph = ipv6_hdr(skb);
        struct xfrm_offload *xo = xfrm_offload(skb);
        u8 proto = iph->nexthdr;

        skb_push(skb, -skb_network_offset(skb));

        if (x->outer_mode.encap == XFRM_MODE_TRANSPORT) {
                __be16 frag;

                ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &proto, &frag);
        }

        esph = ip_esp_hdr(skb);
        *skb_mac_header(skb) = IPPROTO_ESP;

        esph->spi = x->id.spi;
        esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);

        xo->proto = proto;
}

static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x,
                                                struct sk_buff *skb,
                                                netdev_features_t features)
{
        __be16 type = x->inner_mode.family == AF_INET ? htons(ETH_P_IP)
                                                      : htons(ETH_P_IPV6);

        return skb_eth_gso_segment(skb, features, type);
}

static struct sk_buff *xfrm6_transport_gso_segment(struct xfrm_state *x,
                                                   struct sk_buff *skb,
                                                   netdev_features_t features)
{
        const struct net_offload *ops;
        struct sk_buff *segs = ERR_PTR(-EINVAL);
        struct xfrm_offload *xo = xfrm_offload(skb);

        skb->transport_header += x->props.header_len;
        ops = rcu_dereference(inet6_offloads[xo->proto]);
        if (likely(ops && ops->callbacks.gso_segment))
                segs = ops->callbacks.gso_segment(skb, features);

        return segs;
}

static struct sk_buff *xfrm6_beet_gso_segment(struct xfrm_state *x,
                                              struct sk_buff *skb,
                                              netdev_features_t features)
{
        struct xfrm_offload *xo = xfrm_offload(skb);
        struct sk_buff *segs = ERR_PTR(-EINVAL);
        const struct net_offload *ops;
        u8 proto = xo->proto;

        skb->transport_header += x->props.header_len;

        if (x->sel.family != AF_INET6) {
                skb->transport_header -=
                        (sizeof(struct ipv6hdr) - sizeof(struct iphdr));

                if (proto == IPPROTO_BEETPH) {
                        struct ip_beet_phdr *ph =
                                (struct ip_beet_phdr *)skb->data;

                        skb->transport_header += ph->hdrlen * 8;
                        proto = ph->nexthdr;
                } else {
                        skb->transport_header -= IPV4_BEET_PHMAXLEN;
                }

                if (proto == IPPROTO_TCP)
                        skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
        } else {
                __be16 frag;

                skb->transport_header +=
                        ipv6_skip_exthdr(skb, 0, &proto, &frag);
        }

        if (proto == IPPROTO_IPIP)
                skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;

        __skb_pull(skb, skb_transport_offset(skb));
        ops = rcu_dereference(inet6_offloads[proto]);
        if (likely(ops && ops->callbacks.gso_segment))
                segs = ops->callbacks.gso_segment(skb, features);

        return segs;
}

static struct sk_buff *xfrm6_outer_mode_gso_segment(struct xfrm_state *x,
                                                    struct sk_buff *skb,
                                                    netdev_features_t features)
{
        switch (x->outer_mode.encap) {
        case XFRM_MODE_TUNNEL:
                return xfrm6_tunnel_gso_segment(x, skb, features);
        case XFRM_MODE_TRANSPORT:
                return xfrm6_transport_gso_segment(x, skb, features);
        case XFRM_MODE_BEET:
                return xfrm6_beet_gso_segment(x, skb, features);
        }

        return ERR_PTR(-EOPNOTSUPP);
}

static struct sk_buff *esp6_gso_segment(struct sk_buff *skb,
                                        netdev_features_t features)
{
        struct xfrm_state *x;
        struct ip_esp_hdr *esph;
        struct crypto_aead *aead;
        netdev_features_t esp_features = features;
        struct xfrm_offload *xo = xfrm_offload(skb);
        struct sec_path *sp;

        if (!xo)
                return ERR_PTR(-EINVAL);

        if (!(skb_shinfo(skb)->gso_type & SKB_GSO_ESP))
                return ERR_PTR(-EINVAL);

        sp = skb_sec_path(skb);
        x = sp->xvec[sp->len - 1];
        aead = x->data;
        esph = ip_esp_hdr(skb);

        if (esph->spi != x->id.spi)
                return ERR_PTR(-EINVAL);

        if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)))
                return ERR_PTR(-EINVAL);

        __skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead));

        skb->encap_hdr_csum = 1;

        if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev)
                esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK |
                                            NETIF_F_SCTP_CRC);
        else if (!(features & NETIF_F_HW_ESP_TX_CSUM))
                esp_features = features & ~(NETIF_F_CSUM_MASK |
                                            NETIF_F_SCTP_CRC);

        xo->flags |= XFRM_GSO_SEGMENT;

        return xfrm6_outer_mode_gso_segment(x, skb, esp_features);
}

static int esp6_input_tail(struct xfrm_state *x, struct sk_buff *skb)
{
        struct crypto_aead *aead = x->data;
        struct xfrm_offload *xo = xfrm_offload(skb);

        if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead)))
                return -EINVAL;

        if (!(xo->flags & CRYPTO_DONE))
                skb->ip_summed = CHECKSUM_NONE;

        return esp6_input_done2(skb, 0);
}

static int esp6_xmit(struct xfrm_state *x, struct sk_buff *skb,  netdev_features_t features)
{
        int len;
        int err;
        int alen;
        int blksize;
        struct xfrm_offload *xo;
        struct crypto_aead *aead;
        struct esp_info esp;
        bool hw_offload = true;
        __u32 seq;

        esp.inplace = true;

        xo = xfrm_offload(skb);

        if (!xo)
                return -EINVAL;

        if (!(features & NETIF_F_HW_ESP) || x->xso.dev != skb->dev) {
                xo->flags |= CRYPTO_FALLBACK;
                hw_offload = false;
        }

        esp.proto = xo->proto;

        /* skb is pure payload to encrypt */

        aead = x->data;
        alen = crypto_aead_authsize(aead);

        esp.tfclen = 0;
        /* XXX: Add support for tfc padding here. */

        blksize = ALIGN(crypto_aead_blocksize(aead), 4);
        esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
        esp.plen = esp.clen - skb->len - esp.tfclen;
        esp.tailen = esp.tfclen + esp.plen + alen;

        if (!hw_offload || !skb_is_gso(skb)) {
                esp.nfrags = esp6_output_head(x, skb, &esp);
                if (esp.nfrags < 0)
                        return esp.nfrags;
        }

        seq = xo->seq.low;

        esp.esph = ip_esp_hdr(skb);
        esp.esph->spi = x->id.spi;

        skb_push(skb, -skb_network_offset(skb));

        if (xo->flags & XFRM_GSO_SEGMENT) {
                esp.esph->seq_no = htonl(seq);

                if (!skb_is_gso(skb))
                        xo->seq.low++;
                else
                        xo->seq.low += skb_shinfo(skb)->gso_segs;
        }

        if (xo->seq.low < seq)
                xo->seq.hi++;

        esp.seqno = cpu_to_be64(xo->seq.low + ((u64)xo->seq.hi << 32));

        len = skb->len - sizeof(struct ipv6hdr);
        if (len > IPV6_MAXPLEN)
                len = 0;

        ipv6_hdr(skb)->payload_len = htons(len);

        if (hw_offload) {
                if (!skb_ext_add(skb, SKB_EXT_SEC_PATH))
                        return -ENOMEM;

                xo = xfrm_offload(skb);
                if (!xo)
                        return -EINVAL;

                xo->flags |= XFRM_XMIT;
                return 0;
        }

        err = esp6_output_tail(x, skb, &esp);
        if (err)
                return err;

        secpath_reset(skb);

        if (skb_needs_linearize(skb, skb->dev->features) &&
            __skb_linearize(skb))
                return -ENOMEM;
        return 0;
}

static const struct net_offload esp6_offload = {
        .callbacks = {
                .gro_receive = esp6_gro_receive,
                .gso_segment = esp6_gso_segment,
        },
};

static const struct xfrm_type_offload esp6_type_offload = {
        .owner                = THIS_MODULE,
        .proto                     = IPPROTO_ESP,
        .input_tail        = esp6_input_tail,
        .xmit                = esp6_xmit,
        .encap                = esp6_gso_encap,
};

static int __init esp6_offload_init(void)
{
        if (xfrm_register_type_offload(&esp6_type_offload, AF_INET6) < 0) {
                pr_info("%s: can't add xfrm type offload\n", __func__);
                return -EAGAIN;
        }

        return inet6_add_offload(&esp6_offload, IPPROTO_ESP);
}

static void __exit esp6_offload_exit(void)
{
        xfrm_unregister_type_offload(&esp6_type_offload, AF_INET6);
        inet6_del_offload(&esp6_offload, IPPROTO_ESP);
}

module_init(esp6_offload_init);
module_exit(esp6_offload_exit);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET6, XFRM_PROTO_ESP);
MODULE_DESCRIPTION("IPV6 GSO/GRO offload support");

























































































  199 


  199 




























  199 




  199 




  199 




  199 

  198 






















































































































































   20 

























   12 








    7 








   12 







   12 



   12 







 5336 


 5339 

 5334 









    4 




   22 


    5 


   17 

   16 
    2 
   17 


















  199 






  199 


























  199 







  199 





  150 















































































  486 
  487 











    6 



    6 




  491 





  300 















  199 









  150 
























 1625 




    2 
 1625 

 1629 

























































































































   34 


   34 
   34 










    6 


    6 







   22 


    6 


    4 

    4 
    8 



   16 




   31 





   32 

    9 



    2 
   22 


   32 




























  199 


















   10 








    1 


    1 





    2 


    5 


    1 



    5 

















    2 








    2 






















   19 














   19 



















   10 






    3 



    3 









    1 









   16 












   16 
   13 


    3 


    5 







    1 



    4 
    6 




    3 




    1 




















    3 





    4 

    4 














    2 







    1 
    1 

    2 


















    3 



    5 







    1 




    1 



    4 







    2 


   10 














    1 
    9 
    1 



    3 

    3 



   10 
    1 





   14 


   14 














   14 


























































































































































































































































































































    3 






    3 






    4 
    4 



    8 



    2 



    5 
    6 




    1 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
// SPDX-License-Identifier: GPL-2.0-only
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/workqueue.h>
#include <linux/rtnetlink.h>
#include <linux/cache.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <linux/delay.h>
#include <linux/sched.h>
#include <linux/idr.h>
#include <linux/rculist.h>
#include <linux/nsproxy.h>
#include <linux/fs.h>
#include <linux/proc_ns.h>
#include <linux/file.h>
#include <linux/export.h>
#include <linux/user_namespace.h>
#include <linux/net_namespace.h>
#include <linux/sched/task.h>
#include <linux/uidgid.h>
#include <linux/proc_fs.h>
#include <linux/nstree.h>

#include <net/aligned_data.h>
#include <net/sock.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>

/*
 *        Our network namespace constructor/destructor lists
 */

static LIST_HEAD(pernet_list);
static struct list_head *first_device = &pernet_list;

LIST_HEAD(net_namespace_list);
EXPORT_SYMBOL_GPL(net_namespace_list);

/* Protects net_namespace_list. Nests iside rtnl_lock() */
DECLARE_RWSEM(net_rwsem);
EXPORT_SYMBOL_GPL(net_rwsem);

#ifdef CONFIG_KEYS
static struct key_tag init_net_key_domain = { .usage = REFCOUNT_INIT(1) };
#endif

struct net init_net;
EXPORT_SYMBOL(init_net);

static bool init_net_initialized;
/*
 * pernet_ops_rwsem: protects: pernet_list, net_generic_ids,
 * init_net_initialized and first_device pointer.
 * This is internal net namespace object. Please, don't use it
 * outside.
 */
DECLARE_RWSEM(pernet_ops_rwsem);

#define MIN_PERNET_OPS_ID        \
        ((sizeof(struct net_generic) + sizeof(void *) - 1) / sizeof(void *))

#define INITIAL_NET_GEN_PTRS        13 /* +1 for len +2 for rcu_head */

static unsigned int max_gen_ptrs = INITIAL_NET_GEN_PTRS;

static struct net_generic *net_alloc_generic(void)
{
        unsigned int gen_ptrs = READ_ONCE(max_gen_ptrs);
        unsigned int generic_size;
        struct net_generic *ng;

        generic_size = offsetof(struct net_generic, ptr[gen_ptrs]);

        ng = kzalloc(generic_size, GFP_KERNEL);
        if (ng)
                ng->s.len = gen_ptrs;

        return ng;
}

static int net_assign_generic(struct net *net, unsigned int id, void *data)
{
        struct net_generic *ng, *old_ng;

        BUG_ON(id < MIN_PERNET_OPS_ID);

        old_ng = rcu_dereference_protected(net->gen,
                                           lockdep_is_held(&pernet_ops_rwsem));
        if (old_ng->s.len > id) {
                old_ng->ptr[id] = data;
                return 0;
        }

        ng = net_alloc_generic();
        if (!ng)
                return -ENOMEM;

        /*
         * Some synchronisation notes:
         *
         * The net_generic explores the net->gen array inside rcu
         * read section. Besides once set the net->gen->ptr[x]
         * pointer never changes (see rules in netns/generic.h).
         *
         * That said, we simply duplicate this array and schedule
         * the old copy for kfree after a grace period.
         */

        memcpy(&ng->ptr[MIN_PERNET_OPS_ID], &old_ng->ptr[MIN_PERNET_OPS_ID],
               (old_ng->s.len - MIN_PERNET_OPS_ID) * sizeof(void *));
        ng->ptr[id] = data;

        rcu_assign_pointer(net->gen, ng);
        kfree_rcu(old_ng, s.rcu);
        return 0;
}

static int ops_init(const struct pernet_operations *ops, struct net *net)
{
        struct net_generic *ng;
        int err = -ENOMEM;
        void *data = NULL;

        if (ops->id) {
                data = kzalloc(ops->size, GFP_KERNEL);
                if (!data)
                        goto out;

                err = net_assign_generic(net, *ops->id, data);
                if (err)
                        goto cleanup;
        }
        err = 0;
        if (ops->init)
                err = ops->init(net);
        if (!err)
                return 0;

        if (ops->id) {
                ng = rcu_dereference_protected(net->gen,
                                               lockdep_is_held(&pernet_ops_rwsem));
                ng->ptr[*ops->id] = NULL;
        }

cleanup:
        kfree(data);

out:
        return err;
}

static void ops_pre_exit_list(const struct pernet_operations *ops,
                              struct list_head *net_exit_list)
{
        struct net *net;

        if (ops->pre_exit) {
                list_for_each_entry(net, net_exit_list, exit_list)
                        ops->pre_exit(net);
        }
}

static void ops_exit_rtnl_list(const struct list_head *ops_list,
                               const struct pernet_operations *ops,
                               struct list_head *net_exit_list)
{
        const struct pernet_operations *saved_ops = ops;
        LIST_HEAD(dev_kill_list);
        struct net *net;

        rtnl_lock();

        list_for_each_entry(net, net_exit_list, exit_list) {
                __rtnl_net_lock(net);

                ops = saved_ops;
                list_for_each_entry_continue_reverse(ops, ops_list, list) {
                        if (ops->exit_rtnl)
                                ops->exit_rtnl(net, &dev_kill_list);
                }

                __rtnl_net_unlock(net);
        }

        unregister_netdevice_many(&dev_kill_list);

        rtnl_unlock();
}

static void ops_exit_list(const struct pernet_operations *ops,
                          struct list_head *net_exit_list)
{
        if (ops->exit) {
                struct net *net;

                list_for_each_entry(net, net_exit_list, exit_list) {
                        ops->exit(net);
                        cond_resched();
                }
        }

        if (ops->exit_batch)
                ops->exit_batch(net_exit_list);
}

static void ops_free_list(const struct pernet_operations *ops,
                          struct list_head *net_exit_list)
{
        struct net *net;

        if (ops->id) {
                list_for_each_entry(net, net_exit_list, exit_list)
                        kfree(net_generic(net, *ops->id));
        }
}

static void ops_undo_list(const struct list_head *ops_list,
                          const struct pernet_operations *ops,
                          struct list_head *net_exit_list,
                          bool expedite_rcu)
{
        const struct pernet_operations *saved_ops;
        bool hold_rtnl = false;

        if (!ops)
                ops = list_entry(ops_list, typeof(*ops), list);

        saved_ops = ops;

        list_for_each_entry_continue_reverse(ops, ops_list, list) {
                hold_rtnl |= !!ops->exit_rtnl;
                ops_pre_exit_list(ops, net_exit_list);
        }

        /* Another CPU might be rcu-iterating the list, wait for it.
         * This needs to be before calling the exit() notifiers, so the
         * rcu_barrier() after ops_undo_list() isn't sufficient alone.
         * Also the pre_exit() and exit() methods need this barrier.
         */
        if (expedite_rcu)
                synchronize_rcu_expedited();
        else
                synchronize_rcu();

        if (hold_rtnl)
                ops_exit_rtnl_list(ops_list, saved_ops, net_exit_list);

        ops = saved_ops;
        list_for_each_entry_continue_reverse(ops, ops_list, list)
                ops_exit_list(ops, net_exit_list);

        ops = saved_ops;
        list_for_each_entry_continue_reverse(ops, ops_list, list)
                ops_free_list(ops, net_exit_list);
}

static void ops_undo_single(struct pernet_operations *ops,
                            struct list_head *net_exit_list)
{
        LIST_HEAD(ops_list);

        list_add(&ops->list, &ops_list);
        ops_undo_list(&ops_list, NULL, net_exit_list, false);
        list_del(&ops->list);
}

/* should be called with nsid_lock held */
static int alloc_netid(struct net *net, struct net *peer, int reqid)
{
        int min = 0, max = 0;

        if (reqid >= 0) {
                min = reqid;
                max = reqid + 1;
        }

        return idr_alloc(&net->netns_ids, peer, min, max, GFP_ATOMIC);
}

/* This function is used by idr_for_each(). If net is equal to peer, the
 * function returns the id so that idr_for_each() stops. Because we cannot
 * returns the id 0 (idr_for_each() will not stop), we return the magic value
 * NET_ID_ZERO (-1) for it.
 */
#define NET_ID_ZERO -1
static int net_eq_idr(int id, void *net, void *peer)
{
        if (net_eq(net, peer))
                return id ? : NET_ID_ZERO;
        return 0;
}

/* Must be called from RCU-critical section or with nsid_lock held */
static int __peernet2id(const struct net *net, struct net *peer)
{
        int id = idr_for_each(&net->netns_ids, net_eq_idr, peer);

        /* Magic value for id 0. */
        if (id == NET_ID_ZERO)
                return 0;
        if (id > 0)
                return id;

        return NETNSA_NSID_NOT_ASSIGNED;
}

static void rtnl_net_notifyid(struct net *net, int cmd, int id, u32 portid,
                              struct nlmsghdr *nlh, gfp_t gfp);
/* This function returns the id of a peer netns. If no id is assigned, one will
 * be allocated and returned.
 */
int peernet2id_alloc(struct net *net, struct net *peer, gfp_t gfp)
{
        int id;

        if (!check_net(net))
                return NETNSA_NSID_NOT_ASSIGNED;

        spin_lock(&net->nsid_lock);
        id = __peernet2id(net, peer);
        if (id >= 0) {
                spin_unlock(&net->nsid_lock);
                return id;
        }

        /* When peer is obtained from RCU lists, we may race with
         * its cleanup. Check whether it's alive, and this guarantees
         * we never hash a peer back to net->netns_ids, after it has
         * just been idr_remove()'d from there in cleanup_net().
         */
        if (!maybe_get_net(peer)) {
                spin_unlock(&net->nsid_lock);
                return NETNSA_NSID_NOT_ASSIGNED;
        }

        id = alloc_netid(net, peer, -1);
        spin_unlock(&net->nsid_lock);

        put_net(peer);
        if (id < 0)
                return NETNSA_NSID_NOT_ASSIGNED;

        rtnl_net_notifyid(net, RTM_NEWNSID, id, 0, NULL, gfp);

        return id;
}
EXPORT_SYMBOL_GPL(peernet2id_alloc);

/* This function returns, if assigned, the id of a peer netns. */
int peernet2id(const struct net *net, struct net *peer)
{
        int id;

        rcu_read_lock();
        id = __peernet2id(net, peer);
        rcu_read_unlock();

        return id;
}
EXPORT_SYMBOL(peernet2id);

/* This function returns true is the peer netns has an id assigned into the
 * current netns.
 */
bool peernet_has_id(const struct net *net, struct net *peer)
{
        return peernet2id(net, peer) >= 0;
}

struct net *get_net_ns_by_id(const struct net *net, int id)
{
        struct net *peer;

        if (id < 0)
                return NULL;

        rcu_read_lock();
        peer = idr_find(&net->netns_ids, id);
        if (peer)
                peer = maybe_get_net(peer);
        rcu_read_unlock();

        return peer;
}
EXPORT_SYMBOL_GPL(get_net_ns_by_id);

static __net_init void preinit_net_sysctl(struct net *net)
{
        net->core.sysctl_somaxconn = SOMAXCONN;
        /* Limits per socket sk_omem_alloc usage.
         * TCP zerocopy regular usage needs 128 KB.
         */
        net->core.sysctl_optmem_max = 128 * 1024;
        net->core.sysctl_txrehash = SOCK_TXREHASH_ENABLED;
        net->core.sysctl_tstamp_allow_data = 1;
}

/* init code that must occur even if setup_net() is not called. */
static __net_init int preinit_net(struct net *net, struct user_namespace *user_ns)
{
        int ret;

        ret = ns_common_init(net);
        if (ret)
                return ret;

        refcount_set(&net->passive, 1);
        ref_tracker_dir_init(&net->refcnt_tracker, 128, "net_refcnt");
        ref_tracker_dir_init(&net->notrefcnt_tracker, 128, "net_notrefcnt");

        get_random_bytes(&net->hash_mix, sizeof(u32));
        net->dev_base_seq = 1;
        net->user_ns = user_ns;

        idr_init(&net->netns_ids);
        spin_lock_init(&net->nsid_lock);
        mutex_init(&net->ipv4.ra_mutex);

#ifdef CONFIG_DEBUG_NET_SMALL_RTNL
        mutex_init(&net->rtnl_mutex);
        lock_set_cmp_fn(&net->rtnl_mutex, rtnl_net_lock_cmp_fn, NULL);
#endif

        INIT_LIST_HEAD(&net->ptype_all);
        INIT_LIST_HEAD(&net->ptype_specific);
        preinit_net_sysctl(net);
        return 0;
}

/*
 * setup_net runs the initializers for the network namespace object.
 */
static __net_init int setup_net(struct net *net)
{
        /* Must be called with pernet_ops_rwsem held */
        const struct pernet_operations *ops;
        LIST_HEAD(net_exit_list);
        int error = 0;

        net->net_cookie = ns_tree_gen_id(&net->ns);

        list_for_each_entry(ops, &pernet_list, list) {
                error = ops_init(ops, net);
                if (error < 0)
                        goto out_undo;
        }
        down_write(&net_rwsem);
        list_add_tail_rcu(&net->list, &net_namespace_list);
        up_write(&net_rwsem);
        ns_tree_add_raw(net);
out:
        return error;

out_undo:
        /* Walk through the list backwards calling the exit functions
         * for the pernet modules whose init functions did not fail.
         */
        list_add(&net->exit_list, &net_exit_list);
        ops_undo_list(&pernet_list, ops, &net_exit_list, false);
        rcu_barrier();
        goto out;
}

#ifdef CONFIG_NET_NS
static struct ucounts *inc_net_namespaces(struct user_namespace *ns)
{
        return inc_ucount(ns, current_euid(), UCOUNT_NET_NAMESPACES);
}

static void dec_net_namespaces(struct ucounts *ucounts)
{
        dec_ucount(ucounts, UCOUNT_NET_NAMESPACES);
}

static struct kmem_cache *net_cachep __ro_after_init;
static struct workqueue_struct *netns_wq;

static struct net *net_alloc(void)
{
        struct net *net = NULL;
        struct net_generic *ng;

        ng = net_alloc_generic();
        if (!ng)
                goto out;

        net = kmem_cache_zalloc(net_cachep, GFP_KERNEL);
        if (!net)
                goto out_free;

#ifdef CONFIG_KEYS
        net->key_domain = kzalloc(sizeof(struct key_tag), GFP_KERNEL);
        if (!net->key_domain)
                goto out_free_2;
        refcount_set(&net->key_domain->usage, 1);
#endif

        rcu_assign_pointer(net->gen, ng);
out:
        return net;

#ifdef CONFIG_KEYS
out_free_2:
        kmem_cache_free(net_cachep, net);
        net = NULL;
#endif
out_free:
        kfree(ng);
        goto out;
}

static LLIST_HEAD(defer_free_list);

static void net_complete_free(void)
{
        struct llist_node *kill_list;
        struct net *net, *next;

        /* Get the list of namespaces to free from last round. */
        kill_list = llist_del_all(&defer_free_list);

        llist_for_each_entry_safe(net, next, kill_list, defer_free_list)
                kmem_cache_free(net_cachep, net);

}

void net_passive_dec(struct net *net)
{
        if (refcount_dec_and_test(&net->passive)) {
                kfree(rcu_access_pointer(net->gen));

                /* There should not be any trackers left there. */
                ref_tracker_dir_exit(&net->notrefcnt_tracker);

                /* Wait for an extra rcu_barrier() before final free. */
                llist_add(&net->defer_free_list, &defer_free_list);
        }
}

void net_drop_ns(void *p)
{
        struct net *net = (struct net *)p;

        if (net)
                net_passive_dec(net);
}

struct net *copy_net_ns(u64 flags,
                        struct user_namespace *user_ns, struct net *old_net)
{
        struct ucounts *ucounts;
        struct net *net;
        int rv;

        if (!(flags & CLONE_NEWNET))
                return get_net(old_net);

        ucounts = inc_net_namespaces(user_ns);
        if (!ucounts)
                return ERR_PTR(-ENOSPC);

        net = net_alloc();
        if (!net) {
                rv = -ENOMEM;
                goto dec_ucounts;
        }

        rv = preinit_net(net, user_ns);
        if (rv < 0)
                goto dec_ucounts;
        net->ucounts = ucounts;
        get_user_ns(user_ns);

        rv = down_read_killable(&pernet_ops_rwsem);
        if (rv < 0)
                goto put_userns;

        rv = setup_net(net);

        up_read(&pernet_ops_rwsem);

        if (rv < 0) {
put_userns:
                ns_common_free(net);
#ifdef CONFIG_KEYS
                key_remove_domain(net->key_domain);
#endif
                put_user_ns(user_ns);
                net_passive_dec(net);
dec_ucounts:
                dec_net_namespaces(ucounts);
                return ERR_PTR(rv);
        }
        return net;
}

/**
 * net_ns_get_ownership - get sysfs ownership data for @net
 * @net: network namespace in question (can be NULL)
 * @uid: kernel user ID for sysfs objects
 * @gid: kernel group ID for sysfs objects
 *
 * Returns the uid/gid pair of root in the user namespace associated with the
 * given network namespace.
 */
void net_ns_get_ownership(const struct net *net, kuid_t *uid, kgid_t *gid)
{
        if (net) {
                kuid_t ns_root_uid = make_kuid(net->user_ns, 0);
                kgid_t ns_root_gid = make_kgid(net->user_ns, 0);

                if (uid_valid(ns_root_uid))
                        *uid = ns_root_uid;

                if (gid_valid(ns_root_gid))
                        *gid = ns_root_gid;
        } else {
                *uid = GLOBAL_ROOT_UID;
                *gid = GLOBAL_ROOT_GID;
        }
}
EXPORT_SYMBOL_GPL(net_ns_get_ownership);

static void unhash_nsid(struct net *net, struct net *last)
{
        struct net *tmp;
        /* This function is only called from cleanup_net() work,
         * and this work is the only process, that may delete
         * a net from net_namespace_list. So, when the below
         * is executing, the list may only grow. Thus, we do not
         * use for_each_net_rcu() or net_rwsem.
         */
        for_each_net(tmp) {
                int id;

                spin_lock(&tmp->nsid_lock);
                id = __peernet2id(tmp, net);
                if (id >= 0)
                        idr_remove(&tmp->netns_ids, id);
                spin_unlock(&tmp->nsid_lock);
                if (id >= 0)
                        rtnl_net_notifyid(tmp, RTM_DELNSID, id, 0, NULL,
                                          GFP_KERNEL);
                if (tmp == last)
                        break;
        }
        spin_lock(&net->nsid_lock);
        idr_destroy(&net->netns_ids);
        spin_unlock(&net->nsid_lock);
}

static LLIST_HEAD(cleanup_list);

struct task_struct *cleanup_net_task;

static void cleanup_net(struct work_struct *work)
{
        struct llist_node *net_kill_list;
        struct net *net, *tmp, *last;
        LIST_HEAD(net_exit_list);

        WRITE_ONCE(cleanup_net_task, current);

        /* Atomically snapshot the list of namespaces to cleanup */
        net_kill_list = llist_del_all(&cleanup_list);

        down_read(&pernet_ops_rwsem);

        /* Don't let anyone else find us. */
        down_write(&net_rwsem);
        llist_for_each_entry(net, net_kill_list, cleanup_list) {
                ns_tree_remove(net);
                list_del_rcu(&net->list);
        }
        /* Cache last net. After we unlock rtnl, no one new net
         * added to net_namespace_list can assign nsid pointer
         * to a net from net_kill_list (see peernet2id_alloc()).
         * So, we skip them in unhash_nsid().
         *
         * Note, that unhash_nsid() does not delete nsid links
         * between net_kill_list's nets, as they've already
         * deleted from net_namespace_list. But, this would be
         * useless anyway, as netns_ids are destroyed there.
         */
        last = list_last_entry(&net_namespace_list, struct net, list);
        up_write(&net_rwsem);

        llist_for_each_entry(net, net_kill_list, cleanup_list) {
                unhash_nsid(net, last);
                list_add_tail(&net->exit_list, &net_exit_list);
        }

        ops_undo_list(&pernet_list, NULL, &net_exit_list, true);

        up_read(&pernet_ops_rwsem);

        /* Ensure there are no outstanding rcu callbacks using this
         * network namespace.
         */
        rcu_barrier();

        net_complete_free();

        /* Finally it is safe to free my network namespace structure */
        list_for_each_entry_safe(net, tmp, &net_exit_list, exit_list) {
                list_del_init(&net->exit_list);
                ns_common_free(net);
                dec_net_namespaces(net->ucounts);
#ifdef CONFIG_KEYS
                key_remove_domain(net->key_domain);
#endif
                put_user_ns(net->user_ns);
                net_passive_dec(net);
        }
        WRITE_ONCE(cleanup_net_task, NULL);
}

/**
 * net_ns_barrier - wait until concurrent net_cleanup_work is done
 *
 * cleanup_net runs from work queue and will first remove namespaces
 * from the global list, then run net exit functions.
 *
 * Call this in module exit path to make sure that all netns
 * ->exit ops have been invoked before the function is removed.
 */
void net_ns_barrier(void)
{
        down_write(&pernet_ops_rwsem);
        up_write(&pernet_ops_rwsem);
}
EXPORT_SYMBOL(net_ns_barrier);

static DECLARE_WORK(net_cleanup_work, cleanup_net);

void __put_net(struct net *net)
{
        ref_tracker_dir_exit(&net->refcnt_tracker);
        /* Cleanup the network namespace in process context */
        if (llist_add(&net->cleanup_list, &cleanup_list))
                queue_work(netns_wq, &net_cleanup_work);
}
EXPORT_SYMBOL_GPL(__put_net);

/**
 * get_net_ns - increment the refcount of the network namespace
 * @ns: common namespace (net)
 *
 * Returns the net's common namespace or ERR_PTR() if ref is zero.
 */
struct ns_common *get_net_ns(struct ns_common *ns)
{
        struct net *net;

        net = maybe_get_net(container_of(ns, struct net, ns));
        if (net)
                return &net->ns;
        return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL_GPL(get_net_ns);

struct net *get_net_ns_by_fd(int fd)
{
        CLASS(fd, f)(fd);

        if (fd_empty(f))
                return ERR_PTR(-EBADF);

        if (proc_ns_file(fd_file(f))) {
                struct ns_common *ns = get_proc_ns(file_inode(fd_file(f)));
                if (ns->ops == &netns_operations)
                        return get_net(container_of(ns, struct net, ns));
        }

        return ERR_PTR(-EINVAL);
}
EXPORT_SYMBOL_GPL(get_net_ns_by_fd);
#endif

struct net *get_net_ns_by_pid(pid_t pid)
{
        struct task_struct *tsk;
        struct net *net;

        /* Lookup the network namespace */
        net = ERR_PTR(-ESRCH);
        rcu_read_lock();
        tsk = find_task_by_vpid(pid);
        if (tsk) {
                struct nsproxy *nsproxy;
                task_lock(tsk);
                nsproxy = tsk->nsproxy;
                if (nsproxy)
                        net = get_net(nsproxy->net_ns);
                task_unlock(tsk);
        }
        rcu_read_unlock();
        return net;
}
EXPORT_SYMBOL_GPL(get_net_ns_by_pid);

#ifdef CONFIG_NET_NS_REFCNT_TRACKER
static void net_ns_net_debugfs(struct net *net)
{
        ref_tracker_dir_symlink(&net->refcnt_tracker, "netns-%llx-%u-refcnt",
                                net->net_cookie, net->ns.inum);
        ref_tracker_dir_symlink(&net->notrefcnt_tracker, "netns-%llx-%u-notrefcnt",
                                net->net_cookie, net->ns.inum);
}

static int __init init_net_debugfs(void)
{
        ref_tracker_dir_debugfs(&init_net.refcnt_tracker);
        ref_tracker_dir_debugfs(&init_net.notrefcnt_tracker);
        net_ns_net_debugfs(&init_net);
        return 0;
}
late_initcall(init_net_debugfs);
#else
static void net_ns_net_debugfs(struct net *net)
{
}
#endif

static __net_init int net_ns_net_init(struct net *net)
{
        net_ns_net_debugfs(net);
        return 0;
}

static struct pernet_operations __net_initdata net_ns_ops = {
        .init = net_ns_net_init,
};

static const struct nla_policy rtnl_net_policy[NETNSA_MAX + 1] = {
        [NETNSA_NONE]                = { .type = NLA_UNSPEC },
        [NETNSA_NSID]                = { .type = NLA_S32 },
        [NETNSA_PID]                = { .type = NLA_U32 },
        [NETNSA_FD]                = { .type = NLA_U32 },
        [NETNSA_TARGET_NSID]        = { .type = NLA_S32 },
};

static int rtnl_net_newid(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tb[NETNSA_MAX + 1];
        struct nlattr *nla;
        struct net *peer;
        int nsid, err;

        err = nlmsg_parse_deprecated(nlh, sizeof(struct rtgenmsg), tb,
                                     NETNSA_MAX, rtnl_net_policy, extack);
        if (err < 0)
                return err;
        if (!tb[NETNSA_NSID]) {
                NL_SET_ERR_MSG(extack, "nsid is missing");
                return -EINVAL;
        }
        nsid = nla_get_s32(tb[NETNSA_NSID]);

        if (tb[NETNSA_PID]) {
                peer = get_net_ns_by_pid(nla_get_u32(tb[NETNSA_PID]));
                nla = tb[NETNSA_PID];
        } else if (tb[NETNSA_FD]) {
                peer = get_net_ns_by_fd(nla_get_u32(tb[NETNSA_FD]));
                nla = tb[NETNSA_FD];
        } else {
                NL_SET_ERR_MSG(extack, "Peer netns reference is missing");
                return -EINVAL;
        }
        if (IS_ERR(peer)) {
                NL_SET_BAD_ATTR(extack, nla);
                NL_SET_ERR_MSG(extack, "Peer netns reference is invalid");
                return PTR_ERR(peer);
        }

        spin_lock(&net->nsid_lock);
        if (__peernet2id(net, peer) >= 0) {
                spin_unlock(&net->nsid_lock);
                err = -EEXIST;
                NL_SET_BAD_ATTR(extack, nla);
                NL_SET_ERR_MSG(extack,
                               "Peer netns already has a nsid assigned");
                goto out;
        }

        err = alloc_netid(net, peer, nsid);
        spin_unlock(&net->nsid_lock);
        if (err >= 0) {
                rtnl_net_notifyid(net, RTM_NEWNSID, err, NETLINK_CB(skb).portid,
                                  nlh, GFP_KERNEL);
                err = 0;
        } else if (err == -ENOSPC && nsid >= 0) {
                err = -EEXIST;
                NL_SET_BAD_ATTR(extack, tb[NETNSA_NSID]);
                NL_SET_ERR_MSG(extack, "The specified nsid is already used");
        }
out:
        put_net(peer);
        return err;
}

static int rtnl_net_get_size(void)
{
        return NLMSG_ALIGN(sizeof(struct rtgenmsg))
               + nla_total_size(sizeof(s32)) /* NETNSA_NSID */
               + nla_total_size(sizeof(s32)) /* NETNSA_CURRENT_NSID */
               ;
}

struct net_fill_args {
        u32 portid;
        u32 seq;
        int flags;
        int cmd;
        int nsid;
        bool add_ref;
        int ref_nsid;
};

static int rtnl_net_fill(struct sk_buff *skb, struct net_fill_args *args)
{
        struct nlmsghdr *nlh;
        struct rtgenmsg *rth;

        nlh = nlmsg_put(skb, args->portid, args->seq, args->cmd, sizeof(*rth),
                        args->flags);
        if (!nlh)
                return -EMSGSIZE;

        rth = nlmsg_data(nlh);
        rth->rtgen_family = AF_UNSPEC;

        if (nla_put_s32(skb, NETNSA_NSID, args->nsid))
                goto nla_put_failure;

        if (args->add_ref &&
            nla_put_s32(skb, NETNSA_CURRENT_NSID, args->ref_nsid))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int rtnl_net_valid_getid_req(struct sk_buff *skb,
                                    const struct nlmsghdr *nlh,
                                    struct nlattr **tb,
                                    struct netlink_ext_ack *extack)
{
        int i, err;

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(struct rtgenmsg),
                                              tb, NETNSA_MAX, rtnl_net_policy,
                                              extack);

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct rtgenmsg), tb,
                                            NETNSA_MAX, rtnl_net_policy,
                                            extack);
        if (err)
                return err;

        for (i = 0; i <= NETNSA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case NETNSA_PID:
                case NETNSA_FD:
                case NETNSA_NSID:
                case NETNSA_TARGET_NSID:
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "Unsupported attribute in peer netns getid request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int rtnl_net_getid(struct sk_buff *skb, struct nlmsghdr *nlh,
                          struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct nlattr *tb[NETNSA_MAX + 1];
        struct net_fill_args fillargs = {
                .portid = NETLINK_CB(skb).portid,
                .seq = nlh->nlmsg_seq,
                .cmd = RTM_NEWNSID,
        };
        struct net *peer, *target = net;
        struct nlattr *nla;
        struct sk_buff *msg;
        int err;

        err = rtnl_net_valid_getid_req(skb, nlh, tb, extack);
        if (err < 0)
                return err;
        if (tb[NETNSA_PID]) {
                peer = get_net_ns_by_pid(nla_get_u32(tb[NETNSA_PID]));
                nla = tb[NETNSA_PID];
        } else if (tb[NETNSA_FD]) {
                peer = get_net_ns_by_fd(nla_get_u32(tb[NETNSA_FD]));
                nla = tb[NETNSA_FD];
        } else if (tb[NETNSA_NSID]) {
                peer = get_net_ns_by_id(net, nla_get_s32(tb[NETNSA_NSID]));
                if (!peer)
                        peer = ERR_PTR(-ENOENT);
                nla = tb[NETNSA_NSID];
        } else {
                NL_SET_ERR_MSG(extack, "Peer netns reference is missing");
                return -EINVAL;
        }

        if (IS_ERR(peer)) {
                NL_SET_BAD_ATTR(extack, nla);
                NL_SET_ERR_MSG(extack, "Peer netns reference is invalid");
                return PTR_ERR(peer);
        }

        if (tb[NETNSA_TARGET_NSID]) {
                int id = nla_get_s32(tb[NETNSA_TARGET_NSID]);

                target = rtnl_get_net_ns_capable(NETLINK_CB(skb).sk, id);
                if (IS_ERR(target)) {
                        NL_SET_BAD_ATTR(extack, tb[NETNSA_TARGET_NSID]);
                        NL_SET_ERR_MSG(extack,
                                       "Target netns reference is invalid");
                        err = PTR_ERR(target);
                        goto out;
                }
                fillargs.add_ref = true;
                fillargs.ref_nsid = peernet2id(net, peer);
        }

        msg = nlmsg_new(rtnl_net_get_size(), GFP_KERNEL);
        if (!msg) {
                err = -ENOMEM;
                goto out;
        }

        fillargs.nsid = peernet2id(target, peer);
        err = rtnl_net_fill(msg, &fillargs);
        if (err < 0)
                goto err_out;

        err = rtnl_unicast(msg, net, NETLINK_CB(skb).portid);
        goto out;

err_out:
        nlmsg_free(msg);
out:
        if (fillargs.add_ref)
                put_net(target);
        put_net(peer);
        return err;
}

struct rtnl_net_dump_cb {
        struct net *tgt_net;
        struct net *ref_net;
        struct sk_buff *skb;
        struct net_fill_args fillargs;
        int idx;
        int s_idx;
};

/* Runs in RCU-critical section. */
static int rtnl_net_dumpid_one(int id, void *peer, void *data)
{
        struct rtnl_net_dump_cb *net_cb = (struct rtnl_net_dump_cb *)data;
        int ret;

        if (net_cb->idx < net_cb->s_idx)
                goto cont;

        net_cb->fillargs.nsid = id;
        if (net_cb->fillargs.add_ref)
                net_cb->fillargs.ref_nsid = __peernet2id(net_cb->ref_net, peer);
        ret = rtnl_net_fill(net_cb->skb, &net_cb->fillargs);
        if (ret < 0)
                return ret;

cont:
        net_cb->idx++;
        return 0;
}

static int rtnl_valid_dump_net_req(const struct nlmsghdr *nlh, struct sock *sk,
                                   struct rtnl_net_dump_cb *net_cb,
                                   struct netlink_callback *cb)
{
        struct netlink_ext_ack *extack = cb->extack;
        struct nlattr *tb[NETNSA_MAX + 1];
        int err, i;

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct rtgenmsg), tb,
                                            NETNSA_MAX, rtnl_net_policy,
                                            extack);
        if (err < 0)
                return err;

        for (i = 0; i <= NETNSA_MAX; i++) {
                if (!tb[i])
                        continue;

                if (i == NETNSA_TARGET_NSID) {
                        struct net *net;

                        net = rtnl_get_net_ns_capable(sk, nla_get_s32(tb[i]));
                        if (IS_ERR(net)) {
                                NL_SET_BAD_ATTR(extack, tb[i]);
                                NL_SET_ERR_MSG(extack,
                                               "Invalid target network namespace id");
                                return PTR_ERR(net);
                        }
                        net_cb->fillargs.add_ref = true;
                        net_cb->ref_net = net_cb->tgt_net;
                        net_cb->tgt_net = net;
                } else {
                        NL_SET_BAD_ATTR(extack, tb[i]);
                        NL_SET_ERR_MSG(extack,
                                       "Unsupported attribute in dump request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int rtnl_net_dumpid(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct rtnl_net_dump_cb net_cb = {
                .tgt_net = sock_net(skb->sk),
                .skb = skb,
                .fillargs = {
                        .portid = NETLINK_CB(cb->skb).portid,
                        .seq = cb->nlh->nlmsg_seq,
                        .flags = NLM_F_MULTI,
                        .cmd = RTM_NEWNSID,
                },
                .idx = 0,
                .s_idx = cb->args[0],
        };
        int err = 0;

        if (cb->strict_check) {
                err = rtnl_valid_dump_net_req(cb->nlh, skb->sk, &net_cb, cb);
                if (err < 0)
                        goto end;
        }

        rcu_read_lock();
        idr_for_each(&net_cb.tgt_net->netns_ids, rtnl_net_dumpid_one, &net_cb);
        rcu_read_unlock();

        cb->args[0] = net_cb.idx;
end:
        if (net_cb.fillargs.add_ref)
                put_net(net_cb.tgt_net);
        return err;
}

static void rtnl_net_notifyid(struct net *net, int cmd, int id, u32 portid,
                              struct nlmsghdr *nlh, gfp_t gfp)
{
        struct net_fill_args fillargs = {
                .portid = portid,
                .seq = nlh ? nlh->nlmsg_seq : 0,
                .cmd = cmd,
                .nsid = id,
        };
        struct sk_buff *msg;
        int err = -ENOMEM;

        msg = nlmsg_new(rtnl_net_get_size(), gfp);
        if (!msg)
                goto out;

        err = rtnl_net_fill(msg, &fillargs);
        if (err < 0)
                goto err_out;

        rtnl_notify(msg, net, portid, RTNLGRP_NSID, nlh, gfp);
        return;

err_out:
        nlmsg_free(msg);
out:
        rtnl_set_sk_err(net, RTNLGRP_NSID, err);
}

#ifdef CONFIG_NET_NS
static void __init netns_ipv4_struct_check(void)
{
        /* TX readonly hotpath cache lines */
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_early_retrans);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_tso_win_divisor);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_tso_rtt_log);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_autocorking);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_min_snd_mss);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_notsent_lowat);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_limit_output_bytes);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_min_rtt_wlen);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_tcp_wmem);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_tx,
                                      sysctl_ip_fwd_use_pmtu);
        CACHELINE_ASSERT_GROUP_SIZE(struct netns_ipv4, netns_ipv4_read_tx, 33);

        /* TXRX readonly hotpath cache lines */
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_txrx,
                                      sysctl_tcp_moderate_rcvbuf);
        CACHELINE_ASSERT_GROUP_SIZE(struct netns_ipv4, netns_ipv4_read_txrx, 1);

        /* RX readonly hotpath cache line */
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_ip_early_demux);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_tcp_early_demux);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_tcp_l3mdev_accept);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_tcp_reordering);
        CACHELINE_ASSERT_GROUP_MEMBER(struct netns_ipv4, netns_ipv4_read_rx,
                                      sysctl_tcp_rmem);
        CACHELINE_ASSERT_GROUP_SIZE(struct netns_ipv4, netns_ipv4_read_rx, 22);
}
#endif

static const struct rtnl_msg_handler net_ns_rtnl_msg_handlers[] __initconst = {
        {.msgtype = RTM_NEWNSID, .doit = rtnl_net_newid,
         .flags = RTNL_FLAG_DOIT_UNLOCKED},
        {.msgtype = RTM_GETNSID, .doit = rtnl_net_getid,
         .dumpit = rtnl_net_dumpid,
         .flags = RTNL_FLAG_DOIT_UNLOCKED | RTNL_FLAG_DUMP_UNLOCKED},
};

void __init net_ns_init(void)
{
        struct net_generic *ng;

#ifdef CONFIG_NET_NS
        netns_ipv4_struct_check();
        net_cachep = kmem_cache_create("net_namespace", sizeof(struct net),
                                        SMP_CACHE_BYTES,
                                        SLAB_PANIC|SLAB_ACCOUNT, NULL);

        /* Create workqueue for cleanup */
        netns_wq = create_singlethread_workqueue("netns");
        if (!netns_wq)
                panic("Could not create netns workq");
#endif

        ng = net_alloc_generic();
        if (!ng)
                panic("Could not allocate generic netns");

        rcu_assign_pointer(init_net.gen, ng);

#ifdef CONFIG_KEYS
        init_net.key_domain = &init_net_key_domain;
#endif
        /*
         * This currently cannot fail as the initial network namespace
         * has a static inode number.
         */
        if (preinit_net(&init_net, &init_user_ns))
                panic("Could not preinitialize the initial network namespace");

        down_write(&pernet_ops_rwsem);
        if (setup_net(&init_net))
                panic("Could not setup the initial network namespace");

        init_net_initialized = true;
        up_write(&pernet_ops_rwsem);

        if (register_pernet_subsys(&net_ns_ops))
                panic("Could not register network namespace subsystems");

        rtnl_register_many(net_ns_rtnl_msg_handlers);
}

#ifdef CONFIG_NET_NS
static int __register_pernet_operations(struct list_head *list,
                                        struct pernet_operations *ops)
{
        LIST_HEAD(net_exit_list);
        struct net *net;
        int error;

        list_add_tail(&ops->list, list);
        if (ops->init || ops->id) {
                /* We held write locked pernet_ops_rwsem, and parallel
                 * setup_net() and cleanup_net() are not possible.
                 */
                for_each_net(net) {
                        error = ops_init(ops, net);
                        if (error)
                                goto out_undo;
                        list_add_tail(&net->exit_list, &net_exit_list);
                }
        }
        return 0;

out_undo:
        /* If I have an error cleanup all namespaces I initialized */
        list_del(&ops->list);
        ops_undo_single(ops, &net_exit_list);
        return error;
}

static void __unregister_pernet_operations(struct pernet_operations *ops)
{
        LIST_HEAD(net_exit_list);
        struct net *net;

        /* See comment in __register_pernet_operations() */
        for_each_net(net)
                list_add_tail(&net->exit_list, &net_exit_list);

        list_del(&ops->list);
        ops_undo_single(ops, &net_exit_list);
}

#else

static int __register_pernet_operations(struct list_head *list,
                                        struct pernet_operations *ops)
{
        if (!init_net_initialized) {
                list_add_tail(&ops->list, list);
                return 0;
        }

        return ops_init(ops, &init_net);
}

static void __unregister_pernet_operations(struct pernet_operations *ops)
{
        if (!init_net_initialized) {
                list_del(&ops->list);
        } else {
                LIST_HEAD(net_exit_list);

                list_add(&init_net.exit_list, &net_exit_list);
                ops_undo_single(ops, &net_exit_list);
        }
}

#endif /* CONFIG_NET_NS */

static DEFINE_IDA(net_generic_ids);

static int register_pernet_operations(struct list_head *list,
                                      struct pernet_operations *ops)
{
        int error;

        if (WARN_ON(!!ops->id ^ !!ops->size))
                return -EINVAL;

        if (ops->id) {
                error = ida_alloc_min(&net_generic_ids, MIN_PERNET_OPS_ID,
                                GFP_KERNEL);
                if (error < 0)
                        return error;
                *ops->id = error;
                /* This does not require READ_ONCE as writers already hold
                 * pernet_ops_rwsem. But WRITE_ONCE is needed to protect
                 * net_alloc_generic.
                 */
                WRITE_ONCE(max_gen_ptrs, max(max_gen_ptrs, *ops->id + 1));
        }
        error = __register_pernet_operations(list, ops);
        if (error) {
                rcu_barrier();
                if (ops->id)
                        ida_free(&net_generic_ids, *ops->id);
        }

        return error;
}

static void unregister_pernet_operations(struct pernet_operations *ops)
{
        __unregister_pernet_operations(ops);
        rcu_barrier();
        if (ops->id)
                ida_free(&net_generic_ids, *ops->id);
}

/**
 *      register_pernet_subsys - register a network namespace subsystem
 *        @ops:  pernet operations structure for the subsystem
 *
 *        Register a subsystem which has init and exit functions
 *        that are called when network namespaces are created and
 *        destroyed respectively.
 *
 *        When registered all network namespace init functions are
 *        called for every existing network namespace.  Allowing kernel
 *        modules to have a race free view of the set of network namespaces.
 *
 *        When a new network namespace is created all of the init
 *        methods are called in the order in which they were registered.
 *
 *        When a network namespace is destroyed all of the exit methods
 *        are called in the reverse of the order with which they were
 *        registered.
 */
int register_pernet_subsys(struct pernet_operations *ops)
{
        int error;
        down_write(&pernet_ops_rwsem);
        error =  register_pernet_operations(first_device, ops);
        up_write(&pernet_ops_rwsem);
        return error;
}
EXPORT_SYMBOL_GPL(register_pernet_subsys);

/**
 *      unregister_pernet_subsys - unregister a network namespace subsystem
 *        @ops: pernet operations structure to manipulate
 *
 *        Remove the pernet operations structure from the list to be
 *        used when network namespaces are created or destroyed.  In
 *        addition run the exit method for all existing network
 *        namespaces.
 */
void unregister_pernet_subsys(struct pernet_operations *ops)
{
        down_write(&pernet_ops_rwsem);
        unregister_pernet_operations(ops);
        up_write(&pernet_ops_rwsem);
}
EXPORT_SYMBOL_GPL(unregister_pernet_subsys);

/**
 *      register_pernet_device - register a network namespace device
 *        @ops:  pernet operations structure for the subsystem
 *
 *        Register a device which has init and exit functions
 *        that are called when network namespaces are created and
 *        destroyed respectively.
 *
 *        When registered all network namespace init functions are
 *        called for every existing network namespace.  Allowing kernel
 *        modules to have a race free view of the set of network namespaces.
 *
 *        When a new network namespace is created all of the init
 *        methods are called in the order in which they were registered.
 *
 *        When a network namespace is destroyed all of the exit methods
 *        are called in the reverse of the order with which they were
 *        registered.
 */
int register_pernet_device(struct pernet_operations *ops)
{
        int error;
        down_write(&pernet_ops_rwsem);
        error = register_pernet_operations(&pernet_list, ops);
        if (!error && (first_device == &pernet_list))
                first_device = &ops->list;
        up_write(&pernet_ops_rwsem);
        return error;
}
EXPORT_SYMBOL_GPL(register_pernet_device);

/**
 *      unregister_pernet_device - unregister a network namespace netdevice
 *        @ops: pernet operations structure to manipulate
 *
 *        Remove the pernet operations structure from the list to be
 *        used when network namespaces are created or destroyed.  In
 *        addition run the exit method for all existing network
 *        namespaces.
 */
void unregister_pernet_device(struct pernet_operations *ops)
{
        down_write(&pernet_ops_rwsem);
        if (&ops->list == first_device)
                first_device = first_device->next;
        unregister_pernet_operations(ops);
        up_write(&pernet_ops_rwsem);
}
EXPORT_SYMBOL_GPL(unregister_pernet_device);

#ifdef CONFIG_NET_NS
static struct ns_common *netns_get(struct task_struct *task)
{
        struct net *net = NULL;
        struct nsproxy *nsproxy;

        task_lock(task);
        nsproxy = task->nsproxy;
        if (nsproxy)
                net = get_net(nsproxy->net_ns);
        task_unlock(task);

        return net ? &net->ns : NULL;
}

static void netns_put(struct ns_common *ns)
{
        put_net(to_net_ns(ns));
}

static int netns_install(struct nsset *nsset, struct ns_common *ns)
{
        struct nsproxy *nsproxy = nsset->nsproxy;
        struct net *net = to_net_ns(ns);

        if (!ns_capable(net->user_ns, CAP_SYS_ADMIN) ||
            !ns_capable(nsset->cred->user_ns, CAP_SYS_ADMIN))
                return -EPERM;

        put_net(nsproxy->net_ns);
        nsproxy->net_ns = get_net(net);
        return 0;
}

static struct user_namespace *netns_owner(struct ns_common *ns)
{
        return to_net_ns(ns)->user_ns;
}

const struct proc_ns_operations netns_operations = {
        .name                = "net",
        .get                = netns_get,
        .put                = netns_put,
        .install        = netns_install,
        .owner                = netns_owner,
};
#endif








































































































































    5 


    5 
    5 
    5 
    5 
    5 
    5 

    5 





























































































































    5 


    5 



    5 















    5 















    5 



    5 








































































































































































  210 
  210 
  210 

  209 

















































































































































































































































































































































































































































































































































































































































































































































    8 


    8 
    8 

    8 



































































































































































































































































































  479 


  478 



































































  458 



   21 

   21 





























  477 






  479 
  477 





  479 





  478 



  479 


  478 













  479 






















  463 


  463 
  463 








  463 





























































































































   35 

    4 




















   38 




















   40 






   41 
   40 


   41 



   38 
   30 

    9 





   37 



   39 
    9 



    4 
    3 











   35 
   35 
   36 




   15 




    5 












   24 









   41 

































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
// SPDX-License-Identifier: GPL-2.0-only
/*
 * AppArmor security module
 *
 * This file contains AppArmor label definitions
 *
 * Copyright 2017 Canonical Ltd.
 */

#include <linux/audit.h>
#include <linux/seq_file.h>
#include <linux/sort.h>

#include "include/apparmor.h"
#include "include/cred.h"
#include "include/label.h"
#include "include/policy.h"
#include "include/secid.h"


/*
 * the aa_label represents the set of profiles confining an object
 *
 * Labels maintain a reference count to the set of pointers they reference
 * Labels are ref counted by
 *   tasks and object via the security field/security context off the field
 *   code - will take a ref count on a label if it needs the label
 *          beyond what is possible with an rcu_read_lock.
 *   profiles - each profile is a label
 *   secids - a pinned secid will keep a refcount of the label it is
 *          referencing
 *   objects - inode, files, sockets, ...
 *
 * Labels are not ref counted by the label set, so they maybe removed and
 * freed when no longer in use.
 *
 */

#define PROXY_POISON 97
#define LABEL_POISON 100

static void free_proxy(struct aa_proxy *proxy)
{
        if (proxy) {
                /* p->label will not updated any more as p is dead */
                aa_put_label(rcu_dereference_protected(proxy->label, true));
                memset(proxy, 0, sizeof(*proxy));
                RCU_INIT_POINTER(proxy->label, (struct aa_label *)PROXY_POISON);
                kfree(proxy);
        }
}

void aa_proxy_kref(struct kref *kref)
{
        struct aa_proxy *proxy = container_of(kref, struct aa_proxy, count);

        free_proxy(proxy);
}

struct aa_proxy *aa_alloc_proxy(struct aa_label *label, gfp_t gfp)
{
        struct aa_proxy *new;

        new = kzalloc(sizeof(struct aa_proxy), gfp);
        if (new) {
                kref_init(&new->count);
                rcu_assign_pointer(new->label, aa_get_label(label));
        }
        return new;
}

/* requires profile list write lock held */
void __aa_proxy_redirect(struct aa_label *orig, struct aa_label *new)
{
        struct aa_label *tmp;

        AA_BUG(!orig);
        AA_BUG(!new);
        lockdep_assert_held_write(&labels_set(orig)->lock);

        tmp = rcu_dereference_protected(orig->proxy->label,
                                        &labels_ns(orig)->lock);
        rcu_assign_pointer(orig->proxy->label, aa_get_label(new));
        orig->flags |= FLAG_STALE;
        aa_put_label(tmp);
}

static void __proxy_share(struct aa_label *old, struct aa_label *new)
{
        struct aa_proxy *proxy = new->proxy;

        new->proxy = aa_get_proxy(old->proxy);
        __aa_proxy_redirect(old, new);
        aa_put_proxy(proxy);
}


/**
 * ns_cmp - compare ns for label set ordering
 * @a: ns to compare (NOT NULL)
 * @b: ns to compare (NOT NULL)
 *
 * Returns: <0 if a < b
 *          ==0 if a == b
 *          >0  if a > b
 */
static int ns_cmp(struct aa_ns *a, struct aa_ns *b)
{
        int res;

        AA_BUG(!a);
        AA_BUG(!b);
        AA_BUG(!a->base.hname);
        AA_BUG(!b->base.hname);

        if (a == b)
                return 0;

        res = a->level - b->level;
        if (res)
                return res;

        return strcmp(a->base.hname, b->base.hname);
}

/**
 * profile_cmp - profile comparison for set ordering
 * @a: profile to compare (NOT NULL)
 * @b: profile to compare (NOT NULL)
 *
 * Returns: <0  if a < b
 *          ==0 if a == b
 *          >0  if a > b
 */
static int profile_cmp(struct aa_profile *a, struct aa_profile *b)
{
        int res;

        AA_BUG(!a);
        AA_BUG(!b);
        AA_BUG(!a->ns);
        AA_BUG(!b->ns);
        AA_BUG(!a->base.hname);
        AA_BUG(!b->base.hname);

        if (a == b || a->base.hname == b->base.hname)
                return 0;
        res = ns_cmp(a->ns, b->ns);
        if (res)
                return res;

        return strcmp(a->base.hname, b->base.hname);
}

/**
 * vec_cmp - label comparison for set ordering
 * @a: aa_profile to compare (NOT NULL)
 * @an: length of @a
 * @b: aa_profile to compare (NOT NULL)
 * @bn: length of @b
 *
 * Returns: <0  if @a < @b
 *          ==0 if @a == @b
 *          >0  if @a > @b
 */
static int vec_cmp(struct aa_profile **a, int an, struct aa_profile **b, int bn)
{
        int i;

        AA_BUG(!a);
        AA_BUG(!*a);
        AA_BUG(!b);
        AA_BUG(!*b);
        AA_BUG(an <= 0);
        AA_BUG(bn <= 0);

        for (i = 0; i < an && i < bn; i++) {
                int res = profile_cmp(a[i], b[i]);

                if (res != 0)
                        return res;
        }

        return an - bn;
}

static bool vec_is_stale(struct aa_profile **vec, int n)
{
        int i;

        AA_BUG(!vec);

        for (i = 0; i < n; i++) {
                if (profile_is_stale(vec[i]))
                        return true;
        }

        return false;
}

static void accum_label_info(struct aa_label *new)
{
        long u = FLAG_UNCONFINED;
        int i;

        AA_BUG(!new);

        /* size == 1 is a profile and flags must be set as part of creation */
        if (new->size == 1)
                return;

        for (i = 0; i < new->size; i++) {
                u |= new->vec[i]->label.flags & (FLAG_DEBUG1 | FLAG_DEBUG2 |
                                                 FLAG_STALE);
                if (!(u & new->vec[i]->label.flags & FLAG_UNCONFINED))
                        u &= ~FLAG_UNCONFINED;
                new->mediates |= new->vec[i]->label.mediates;
        }
        new->flags |= u;
}

static int sort_cmp(const void *a, const void *b)
{
        return profile_cmp(*(struct aa_profile **)a, *(struct aa_profile **)b);
}

/*
 * assumes vec is sorted
 * Assumes @vec has null terminator at vec[n], and will null terminate
 * vec[n - dups]
 */
static inline int unique(struct aa_profile **vec, int n)
{
        int i, pos, dups = 0;

        AA_BUG(n < 1);
        AA_BUG(!vec);

        pos = 0;
        for (i = 1; i < n; i++) {
                int res = profile_cmp(vec[pos], vec[i]);

                AA_BUG(res > 0, "vec not sorted");
                if (res == 0) {
                        /* drop duplicate */
                        aa_put_profile(vec[i]);
                        dups++;
                        continue;
                }
                pos++;
                if (dups)
                        vec[pos] = vec[i];
        }

        AA_BUG(dups < 0);

        return dups;
}

/**
 * aa_vec_unique - canonical sort and unique a list of profiles
 * @n: number of refcounted profiles in the list (@n > 0)
 * @vec: list of profiles to sort and merge
 * @flags: null terminator flags of @vec
 *
 * Returns: the number of duplicates eliminated == references put
 *
 * If @flags & VEC_FLAG_TERMINATE @vec has null terminator at vec[n], and will
 * null terminate vec[n - dups]
 */
int aa_vec_unique(struct aa_profile **vec, int n, int flags)
{
        int i, dups = 0;

        AA_BUG(n < 1);
        AA_BUG(!vec);

        /* vecs are usually small and inorder, have a fallback for larger */
        if (n > 8) {
                sort(vec, n, sizeof(struct aa_profile *), sort_cmp, NULL);
                dups = unique(vec, n);
                goto out;
        }

        /* insertion sort + unique in one */
        for (i = 1; i < n; i++) {
                struct aa_profile *tmp = vec[i];
                int pos, j;

                for (pos = i - 1 - dups; pos >= 0; pos--) {
                        int res = profile_cmp(vec[pos], tmp);

                        if (res == 0) {
                                /* drop duplicate entry */
                                aa_put_profile(tmp);
                                dups++;
                                goto continue_outer;
                        } else if (res < 0)
                                break;
                }
                /* pos is at entry < tmp, or index -1. Set to insert pos */
                pos++;

                for (j = i - dups; j > pos; j--)
                        vec[j] = vec[j - 1];
                vec[pos] = tmp;
continue_outer:
                ;
        }

        AA_BUG(dups < 0);

out:
        if (flags & VEC_FLAG_TERMINATE)
                vec[n - dups] = NULL;

        return dups;
}


void aa_label_destroy(struct aa_label *label)
{
        AA_BUG(!label);

        if (!label_isprofile(label)) {
                struct aa_profile *profile;
                struct label_it i;

                aa_put_str(label->hname);

                label_for_each(i, label, profile) {
                        aa_put_profile(profile);
                        label->vec[i.i] = (struct aa_profile *)
                                           (LABEL_POISON + (long) i.i);
                }
        }

        if (label->proxy) {
                if (rcu_dereference_protected(label->proxy->label, true) == label)
                        rcu_assign_pointer(label->proxy->label, NULL);
                aa_put_proxy(label->proxy);
        }
        aa_free_secid(label->secid);

        label->proxy = (struct aa_proxy *) PROXY_POISON + 1;
}

void aa_label_free(struct aa_label *label)
{
        if (!label)
                return;

        aa_label_destroy(label);
        kfree(label);
}

static void label_free_switch(struct aa_label *label)
{
        if (label->flags & FLAG_NS_COUNT)
                aa_free_ns(labels_ns(label));
        else if (label_isprofile(label))
                aa_free_profile(labels_profile(label));
        else
                aa_label_free(label);
}

static void label_free_rcu(struct rcu_head *head)
{
        struct aa_label *label = container_of(head, struct aa_label, rcu);

        if (label->flags & FLAG_IN_TREE)
                (void) aa_label_remove(label);
        label_free_switch(label);
}

void aa_label_kref(struct kref *kref)
{
        struct aa_label *label = container_of(kref, struct aa_label, count);
        struct aa_ns *ns = labels_ns(label);

        if (!ns) {
                /* never live, no rcu callback needed, just using the fn */
                label_free_switch(label);
                return;
        }
        /* TODO: update labels_profile macro so it works here */
        AA_BUG(label_isprofile(label) &&
               on_list_rcu(&label->vec[0]->base.profiles));
        AA_BUG(label_isprofile(label) &&
               on_list_rcu(&label->vec[0]->base.list));

        /* TODO: if compound label and not stale add to reclaim cache */
        call_rcu(&label->rcu, label_free_rcu);
}

static void label_free_or_put_new(struct aa_label *label, struct aa_label *new)
{
        if (label != new)
                /* need to free directly to break circular ref with proxy */
                aa_label_free(new);
        else
                aa_put_label(new);
}

bool aa_label_init(struct aa_label *label, int size, gfp_t gfp)
{
        AA_BUG(!label);
        AA_BUG(size < 1);

        if (aa_alloc_secid(label, gfp) < 0)
                return false;

        label->size = size;                        /* doesn't include null */
        label->vec[size] = NULL;                /* null terminate */
        kref_init(&label->count);
        RB_CLEAR_NODE(&label->node);

        return true;
}

/**
 * aa_label_alloc - allocate a label with a profile vector of @size length
 * @size: size of profile vector in the label
 * @proxy: proxy to use OR null if to allocate a new one
 * @gfp: memory allocation type
 *
 * Returns: new label
 *     else NULL if failed
 */
struct aa_label *aa_label_alloc(int size, struct aa_proxy *proxy, gfp_t gfp)
{
        struct aa_label *new;

        AA_BUG(size < 1);

        /*  + 1 for null terminator entry on vec */
        new = kzalloc(struct_size(new, vec, size + 1), gfp);
        AA_DEBUG(DEBUG_LABEL, "%s (%p)\n", __func__, new);
        if (!new)
                goto fail;

        if (!aa_label_init(new, size, gfp))
                goto fail;

        if (!proxy) {
                proxy = aa_alloc_proxy(new, gfp);
                if (!proxy)
                        goto fail;
        } else
                aa_get_proxy(proxy);
        /* just set new's proxy, don't redirect proxy here if it was passed in*/
        new->proxy = proxy;

        return new;

fail:
        kfree(new);

        return NULL;
}


/**
 * label_cmp - label comparison for set ordering
 * @a: label to compare (NOT NULL)
 * @b: label to compare (NOT NULL)
 *
 * Returns: <0  if a < b
 *          ==0 if a == b
 *          >0  if a > b
 */
static int label_cmp(struct aa_label *a, struct aa_label *b)
{
        AA_BUG(!b);

        if (a == b)
                return 0;

        return vec_cmp(a->vec, a->size, b->vec, b->size);
}

/* helper fn for label_for_each_confined */
int aa_label_next_confined(struct aa_label *label, int i)
{
        AA_BUG(!label);
        AA_BUG(i < 0);

        for (; i < label->size; i++) {
                if (!profile_unconfined(label->vec[i]))
                        return i;
        }

        return i;
}

/**
 * __aa_label_next_not_in_set - return the next profile of @sub not in @set
 * @I: label iterator
 * @set: label to test against
 * @sub: label to if is subset of @set
 *
 * Returns: profile in @sub that is not in @set, with iterator set pos after
 *     else NULL if @sub is a subset of @set
 */
struct aa_profile *__aa_label_next_not_in_set(struct label_it *I,
                                              struct aa_label *set,
                                              struct aa_label *sub)
{
        AA_BUG(!set);
        AA_BUG(!I);
        AA_BUG(I->i < 0);
        AA_BUG(I->i > set->size);
        AA_BUG(!sub);
        AA_BUG(I->j < 0);
        AA_BUG(I->j > sub->size);

        while (I->j < sub->size && I->i < set->size) {
                int res = profile_cmp(sub->vec[I->j], set->vec[I->i]);

                if (res == 0) {
                        (I->j)++;
                        (I->i)++;
                } else if (res > 0)
                        (I->i)++;
                else
                        return sub->vec[(I->j)++];
        }

        if (I->j < sub->size)
                return sub->vec[(I->j)++];

        return NULL;
}

/**
 * aa_label_is_subset - test if @sub is a subset of @set
 * @set: label to test against
 * @sub: label to test if is subset of @set
 *
 * Returns: true if @sub is subset of @set
 *     else false
 */
bool aa_label_is_subset(struct aa_label *set, struct aa_label *sub)
{
        struct label_it i = { };

        AA_BUG(!set);
        AA_BUG(!sub);

        if (sub == set)
                return true;

        return __aa_label_next_not_in_set(&i, set, sub) == NULL;
}

/**
 * aa_label_is_unconfined_subset - test if @sub is a subset of @set
 * @set: label to test against
 * @sub: label to test if is subset of @set
 *
 * This checks for subset but taking into account unconfined. IF
 * @sub contains an unconfined profile that does not have a matching
 * unconfined in @set then this will not cause the test to fail.
 * Conversely we don't care about an unconfined in @set that is not in
 * @sub
 *
 * Returns: true if @sub is special_subset of @set
 *     else false
 */
bool aa_label_is_unconfined_subset(struct aa_label *set, struct aa_label *sub)
{
        struct label_it i = { };
        struct aa_profile *p;

        AA_BUG(!set);
        AA_BUG(!sub);

        if (sub == set)
                return true;

        do {
                p = __aa_label_next_not_in_set(&i, set, sub);
                if (p && !profile_unconfined(p))
                        break;
        } while (p);

        return p == NULL;
}


/**
 * __label_remove - remove @label from the label set
 * @label: label to remove
 * @new: label to redirect to
 *
 * Requires: labels_set(@label)->lock write_lock
 * Returns:  true if the label was in the tree and removed
 */
static bool __label_remove(struct aa_label *label, struct aa_label *new)
{
        struct aa_labelset *ls = labels_set(label);

        AA_BUG(!ls);
        AA_BUG(!label);
        lockdep_assert_held_write(&ls->lock);

        if (new)
                __aa_proxy_redirect(label, new);

        if (!label_is_stale(label))
                __label_make_stale(label);

        if (label->flags & FLAG_IN_TREE) {
                rb_erase(&label->node, &ls->root);
                label->flags &= ~FLAG_IN_TREE;
                return true;
        }

        return false;
}

/**
 * __label_replace - replace @old with @new in label set
 * @old: label to remove from label set
 * @new: label to replace @old with
 *
 * Requires: labels_set(@old)->lock write_lock
 *           valid ref count be held on @new
 * Returns: true if @old was in set and replaced by @new
 *
 * Note: current implementation requires label set be order in such a way
 *       that @new directly replaces @old position in the set (ie.
 *       using pointer comparison of the label address would not work)
 */
static bool __label_replace(struct aa_label *old, struct aa_label *new)
{
        struct aa_labelset *ls = labels_set(old);

        AA_BUG(!ls);
        AA_BUG(!old);
        AA_BUG(!new);
        lockdep_assert_held_write(&ls->lock);
        AA_BUG(new->flags & FLAG_IN_TREE);

        if (!label_is_stale(old))
                __label_make_stale(old);

        if (old->flags & FLAG_IN_TREE) {
                rb_replace_node(&old->node, &new->node, &ls->root);
                old->flags &= ~FLAG_IN_TREE;
                new->flags |= FLAG_IN_TREE;
                accum_label_info(new);
                return true;
        }

        return false;
}

/**
 * __label_insert - attempt to insert @l into a label set
 * @ls: set of labels to insert @l into (NOT NULL)
 * @label: new label to insert (NOT NULL)
 * @replace: whether insertion should replace existing entry that is not stale
 *
 * Requires: @ls->lock
 *           caller to hold a valid ref on l
 *           if @replace is true l has a preallocated proxy associated
 * Returns: @l if successful in inserting @l - with additional refcount
 *          else ref counted equivalent label that is already in the set,
 *          the else condition only happens if @replace is false
 */
static struct aa_label *__label_insert(struct aa_labelset *ls,
                                       struct aa_label *label, bool replace)
{
        struct rb_node **new, *parent = NULL;

        AA_BUG(!ls);
        AA_BUG(!label);
        AA_BUG(labels_set(label) != ls);
        lockdep_assert_held_write(&ls->lock);
        AA_BUG(label->flags & FLAG_IN_TREE);

        /* Figure out where to put new node */
        new = &ls->root.rb_node;
        while (*new) {
                struct aa_label *this = rb_entry(*new, struct aa_label, node);
                int result = label_cmp(label, this);

                parent = *new;
                if (result == 0) {
                        /* !__aa_get_label means queued for destruction,
                         * so replace in place, however the label has
                         * died before the replacement so do not share
                         * the proxy
                         */
                        if (!replace && !label_is_stale(this)) {
                                if (__aa_get_label(this))
                                        return this;
                        } else
                                __proxy_share(this, label);
                        AA_BUG(!__label_replace(this, label));
                        return aa_get_label(label);
                } else if (result < 0)
                        new = &((*new)->rb_left);
                else /* (result > 0) */
                        new = &((*new)->rb_right);
        }

        /* Add new node and rebalance tree. */
        rb_link_node(&label->node, parent, new);
        rb_insert_color(&label->node, &ls->root);
        label->flags |= FLAG_IN_TREE;
        accum_label_info(label);

        return aa_get_label(label);
}

/**
 * __vec_find - find label that matches @vec in label set
 * @vec: vec of profiles to find matching label for (NOT NULL)
 * @n: length of @vec
 *
 * Requires: @vec_labelset(vec) lock held
 *           caller to hold a valid ref on l
 *
 * Returns: ref counted @label if matching label is in tree
 *          ref counted label that is equiv to @l in tree
 *     else NULL if @vec equiv is not in tree
 */
static struct aa_label *__vec_find(struct aa_profile **vec, int n)
{
        struct rb_node *node;

        AA_BUG(!vec);
        AA_BUG(!*vec);
        AA_BUG(n <= 0);

        node = vec_labelset(vec, n)->root.rb_node;
        while (node) {
                struct aa_label *this = rb_entry(node, struct aa_label, node);
                int result = vec_cmp(this->vec, this->size, vec, n);

                if (result > 0)
                        node = node->rb_left;
                else if (result < 0)
                        node = node->rb_right;
                else
                        return __aa_get_label(this);
        }

        return NULL;
}

/**
 * __label_find - find label @label in label set
 * @label: label to find (NOT NULL)
 *
 * Requires: labels_set(@label)->lock held
 *           caller to hold a valid ref on l
 *
 * Returns: ref counted @label if @label is in tree OR
 *          ref counted label that is equiv to @label in tree
 *     else NULL if @label or equiv is not in tree
 */
static struct aa_label *__label_find(struct aa_label *label)
{
        AA_BUG(!label);

        return __vec_find(label->vec, label->size);
}


/**
 * aa_label_remove - remove a label from the labelset
 * @label: label to remove
 *
 * Returns: true if @label was removed from the tree
 *     else @label was not in tree so it could not be removed
 */
bool aa_label_remove(struct aa_label *label)
{
        struct aa_labelset *ls = labels_set(label);
        unsigned long flags;
        bool res;

        AA_BUG(!ls);

        write_lock_irqsave(&ls->lock, flags);
        res = __label_remove(label, ns_unconfined(labels_ns(label)));
        write_unlock_irqrestore(&ls->lock, flags);

        return res;
}

/**
 * aa_label_replace - replace a label @old with a new version @new
 * @old: label to replace
 * @new: label replacing @old
 *
 * Returns: true if @old was in tree and replaced
 *     else @old was not in tree, and @new was not inserted
 */
bool aa_label_replace(struct aa_label *old, struct aa_label *new)
{
        unsigned long flags;
        bool res;

        if (name_is_shared(old, new) && labels_ns(old) == labels_ns(new)) {
                write_lock_irqsave(&labels_set(old)->lock, flags);
                if (old->proxy != new->proxy)
                        __proxy_share(old, new);
                else
                        __aa_proxy_redirect(old, new);
                res = __label_replace(old, new);
                write_unlock_irqrestore(&labels_set(old)->lock, flags);
        } else {
                struct aa_label *l;
                struct aa_labelset *ls = labels_set(old);

                write_lock_irqsave(&ls->lock, flags);
                res = __label_remove(old, new);
                if (labels_ns(old) != labels_ns(new)) {
                        write_unlock_irqrestore(&ls->lock, flags);
                        ls = labels_set(new);
                        write_lock_irqsave(&ls->lock, flags);
                }
                l = __label_insert(ls, new, true);
                res = (l == new);
                write_unlock_irqrestore(&ls->lock, flags);
                aa_put_label(l);
        }

        return res;
}

/**
 * vec_find - find label @l in label set
 * @vec: array of profiles to find equiv label for (NOT NULL)
 * @n: length of @vec
 *
 * Returns: refcounted label if @vec equiv is in tree
 *     else NULL if @vec equiv is not in tree
 */
static struct aa_label *vec_find(struct aa_profile **vec, int n)
{
        struct aa_labelset *ls;
        struct aa_label *label;
        unsigned long flags;

        AA_BUG(!vec);
        AA_BUG(!*vec);
        AA_BUG(n <= 0);

        ls = vec_labelset(vec, n);
        read_lock_irqsave(&ls->lock, flags);
        label = __vec_find(vec, n);
        read_unlock_irqrestore(&ls->lock, flags);

        return label;
}

/* requires sort and merge done first */
static struct aa_label *vec_create_and_insert_label(struct aa_profile **vec,
                                                    int len, gfp_t gfp)
{
        struct aa_label *label = NULL;
        struct aa_labelset *ls;
        unsigned long flags;
        struct aa_label *new;
        int i;

        AA_BUG(!vec);

        if (len == 1)
                return aa_get_label(&vec[0]->label);

        ls = labels_set(&vec[len - 1]->label);

        /* TODO: enable when read side is lockless
         * check if label exists before taking locks
         */
        new = aa_label_alloc(len, NULL, gfp);
        if (!new)
                return NULL;

        for (i = 0; i < len; i++)
                new->vec[i] = aa_get_profile(vec[i]);

        write_lock_irqsave(&ls->lock, flags);
        label = __label_insert(ls, new, false);
        write_unlock_irqrestore(&ls->lock, flags);
        label_free_or_put_new(label, new);

        return label;
}

struct aa_label *aa_vec_find_or_create_label(struct aa_profile **vec, int len,
                                             gfp_t gfp)
{
        struct aa_label *label = vec_find(vec, len);

        if (label)
                return label;

        return vec_create_and_insert_label(vec, len, gfp);
}


/**
 * aa_label_insert - insert label @label into @ls or return existing label
 * @ls: labelset to insert @label into
 * @label: label to insert
 *
 * Requires: caller to hold a valid ref on @label
 *
 * Returns: ref counted @label if successful in inserting @label
 *     else ref counted equivalent label that is already in the set
 */
struct aa_label *aa_label_insert(struct aa_labelset *ls, struct aa_label *label)
{
        struct aa_label *l;
        unsigned long flags;

        AA_BUG(!ls);
        AA_BUG(!label);

        /* check if label exists before taking lock */
        if (!label_is_stale(label)) {
                read_lock_irqsave(&ls->lock, flags);
                l = __label_find(label);
                read_unlock_irqrestore(&ls->lock, flags);
                if (l)
                        return l;
        }

        write_lock_irqsave(&ls->lock, flags);
        l = __label_insert(ls, label, false);
        write_unlock_irqrestore(&ls->lock, flags);

        return l;
}


/**
 * aa_label_next_in_merge - find the next profile when merging @a and @b
 * @I: label iterator
 * @a: label to merge
 * @b: label to merge
 *
 * Returns: next profile
 *     else null if no more profiles
 */
struct aa_profile *aa_label_next_in_merge(struct label_it *I,
                                          struct aa_label *a,
                                          struct aa_label *b)
{
        AA_BUG(!a);
        AA_BUG(!b);
        AA_BUG(!I);
        AA_BUG(I->i < 0);
        AA_BUG(I->i > a->size);
        AA_BUG(I->j < 0);
        AA_BUG(I->j > b->size);

        if (I->i < a->size) {
                if (I->j < b->size) {
                        int res = profile_cmp(a->vec[I->i], b->vec[I->j]);

                        if (res > 0)
                                return b->vec[(I->j)++];
                        if (res == 0)
                                (I->j)++;
                }

                return a->vec[(I->i)++];
        }

        if (I->j < b->size)
                return b->vec[(I->j)++];

        return NULL;
}

/**
 * label_merge_cmp - cmp of @a merging with @b against @z for set ordering
 * @a: label to merge then compare (NOT NULL)
 * @b: label to merge then compare (NOT NULL)
 * @z: label to compare merge against (NOT NULL)
 *
 * Assumes: using the most recent versions of @a, @b, and @z
 *
 * Returns: <0  if a < b
 *          ==0 if a == b
 *          >0  if a > b
 */
static int label_merge_cmp(struct aa_label *a, struct aa_label *b,
                           struct aa_label *z)
{
        struct aa_profile *p = NULL;
        struct label_it i = { };
        int k;

        AA_BUG(!a);
        AA_BUG(!b);
        AA_BUG(!z);

        for (k = 0;
             k < z->size && (p = aa_label_next_in_merge(&i, a, b));
             k++) {
                int res = profile_cmp(p, z->vec[k]);

                if (res != 0)
                        return res;
        }

        if (p)
                return 1;
        else if (k < z->size)
                return -1;
        return 0;
}

/**
 * label_merge_insert - create a new label by merging @a and @b
 * @new: preallocated label to merge into (NOT NULL)
 * @a: label to merge with @b  (NOT NULL)
 * @b: label to merge with @a  (NOT NULL)
 *
 * Requires: preallocated proxy
 *
 * Returns: ref counted label either @new if merge is unique
 *          @a if @b is a subset of @a
 *          @b if @a is a subset of @b
 *
 * NOTE: will not use @new if the merge results in @new == @a or @b
 *
 *       Must be used within labelset write lock to avoid racing with
 *       setting labels stale.
 */
static struct aa_label *label_merge_insert(struct aa_label *new,
                                           struct aa_label *a,
                                           struct aa_label *b)
{
        struct aa_label *label;
        struct aa_labelset *ls;
        struct aa_profile *next;
        struct label_it i;
        unsigned long flags;
        int k = 0, invcount = 0;
        bool stale = false;

        AA_BUG(!a);
        AA_BUG(a->size < 0);
        AA_BUG(!b);
        AA_BUG(b->size < 0);
        AA_BUG(!new);
        AA_BUG(new->size < a->size + b->size);

        label_for_each_in_merge(i, a, b, next) {
                AA_BUG(!next);
                if (profile_is_stale(next)) {
                        new->vec[k] = aa_get_newest_profile(next);
                        AA_BUG(!new->vec[k]->label.proxy);
                        AA_BUG(!new->vec[k]->label.proxy->label);
                        if (next->label.proxy != new->vec[k]->label.proxy)
                                invcount++;
                        k++;
                        stale = true;
                } else
                        new->vec[k++] = aa_get_profile(next);
        }
        /* set to actual size which is <= allocated len */
        new->size = k;
        new->vec[k] = NULL;

        if (invcount) {
                new->size -= aa_vec_unique(&new->vec[0], new->size,
                                           VEC_FLAG_TERMINATE);
                /* TODO: deal with reference labels */
                if (new->size == 1) {
                        label = aa_get_label(&new->vec[0]->label);
                        return label;
                }
        } else if (!stale) {
                /*
                 * merge could be same as a || b, note: it is not possible
                 * for new->size == a->size == b->size unless a == b
                 */
                if (k == a->size)
                        return aa_get_label(a);
                else if (k == b->size)
                        return aa_get_label(b);
        }
        ls = labels_set(new);
        write_lock_irqsave(&ls->lock, flags);
        label = __label_insert(labels_set(new), new, false);
        write_unlock_irqrestore(&ls->lock, flags);

        return label;
}

/**
 * labelset_of_merge - find which labelset a merged label should be inserted
 * @a: label to merge and insert
 * @b: label to merge and insert
 *
 * Returns: labelset that the merged label should be inserted into
 */
static struct aa_labelset *labelset_of_merge(struct aa_label *a,
                                             struct aa_label *b)
{
        struct aa_ns *nsa = labels_ns(a);
        struct aa_ns *nsb = labels_ns(b);

        if (ns_cmp(nsa, nsb) <= 0)
                return &nsa->labels;
        return &nsb->labels;
}

/**
 * __label_find_merge - find label that is equiv to merge of @a and @b
 * @ls: set of labels to search (NOT NULL)
 * @a: label to merge with @b  (NOT NULL)
 * @b: label to merge with @a  (NOT NULL)
 *
 * Requires: ls->lock read_lock held
 *
 * Returns: ref counted label that is equiv to merge of @a and @b
 *     else NULL if merge of @a and @b is not in set
 */
static struct aa_label *__label_find_merge(struct aa_labelset *ls,
                                           struct aa_label *a,
                                           struct aa_label *b)
{
        struct rb_node *node;

        AA_BUG(!ls);
        AA_BUG(!a);
        AA_BUG(!b);

        if (a == b)
                return __label_find(a);

        node  = ls->root.rb_node;
        while (node) {
                struct aa_label *this = container_of(node, struct aa_label,
                                                     node);
                int result = label_merge_cmp(a, b, this);

                if (result < 0)
                        node = node->rb_left;
                else if (result > 0)
                        node = node->rb_right;
                else
                        return __aa_get_label(this);
        }

        return NULL;
}


/**
 * aa_label_find_merge - find label that is equiv to merge of @a and @b
 * @a: label to merge with @b  (NOT NULL)
 * @b: label to merge with @a  (NOT NULL)
 *
 * Requires: labels be fully constructed with a valid ns
 *
 * Returns: ref counted label that is equiv to merge of @a and @b
 *     else NULL if merge of @a and @b is not in set
 */
struct aa_label *aa_label_find_merge(struct aa_label *a, struct aa_label *b)
{
        struct aa_labelset *ls;
        struct aa_label *label, *ar = NULL, *br = NULL;
        unsigned long flags;

        AA_BUG(!a);
        AA_BUG(!b);

        if (label_is_stale(a))
                a = ar = aa_get_newest_label(a);
        if (label_is_stale(b))
                b = br = aa_get_newest_label(b);
        ls = labelset_of_merge(a, b);
        read_lock_irqsave(&ls->lock, flags);
        label = __label_find_merge(ls, a, b);
        read_unlock_irqrestore(&ls->lock, flags);
        aa_put_label(ar);
        aa_put_label(br);

        return label;
}

/**
 * aa_label_merge - attempt to insert new merged label of @a and @b
 * @a: label to merge with @b  (NOT NULL)
 * @b: label to merge with @a  (NOT NULL)
 * @gfp: memory allocation type
 *
 * Requires: caller to hold valid refs on @a and @b
 *           labels be fully constructed with a valid ns
 *
 * Returns: ref counted new label if successful in inserting merge of a & b
 *     else ref counted equivalent label that is already in the set.
 *     else NULL if could not create label (-ENOMEM)
 */
struct aa_label *aa_label_merge(struct aa_label *a, struct aa_label *b,
                                gfp_t gfp)
{
        struct aa_label *label = NULL;

        AA_BUG(!a);
        AA_BUG(!b);

        if (a == b)
                return aa_get_newest_label(a);

        /* TODO: enable when read side is lockless
         * check if label exists before taking locks
        if (!label_is_stale(a) && !label_is_stale(b))
                label = aa_label_find_merge(a, b);
        */

        if (!label) {
                struct aa_label *new;

                a = aa_get_newest_label(a);
                b = aa_get_newest_label(b);

                /* could use label_merge_len(a, b), but requires double
                 * comparison for small savings
                 */
                new = aa_label_alloc(a->size + b->size, NULL, gfp);
                if (!new)
                        goto out;

                label = label_merge_insert(new, a, b);
                label_free_or_put_new(label, new);
out:
                aa_put_label(a);
                aa_put_label(b);
        }

        return label;
}

/* match a profile and its associated ns component if needed
 * Assumes visibility test has already been done.
 * If a subns profile is not to be matched should be prescreened with
 * visibility test.
 */
static inline aa_state_t match_component(struct aa_profile *profile,
                                         struct aa_ruleset *rules,
                                         struct aa_profile *tp,
                                         aa_state_t state)
{
        const char *ns_name;

        if (profile->ns == tp->ns)
                return aa_dfa_match(rules->policy->dfa, state, tp->base.hname);

        /* try matching with namespace name and then profile */
        ns_name = aa_ns_name(profile->ns, tp->ns, true);
        state = aa_dfa_match_len(rules->policy->dfa, state, ":", 1);
        state = aa_dfa_match(rules->policy->dfa, state, ns_name);
        state = aa_dfa_match_len(rules->policy->dfa, state, ":", 1);
        return aa_dfa_match(rules->policy->dfa, state, tp->base.hname);
}

/**
 * label_compound_match - find perms for full compound label
 * @profile: profile to find perms for
 * @rules: ruleset to search
 * @label: label to check access permissions for
 * @state: state to start match in
 * @subns: whether to do permission checks on components in a subns
 * @request: permissions to request
 * @perms: perms struct to set
 *
 * Returns: 0 on success else ERROR
 *
 * For the label A//&B//&C this does the perm match for A//&B//&C
 * @perms should be preinitialized with allperms OR a previous permission
 *        check to be stacked.
 */
static int label_compound_match(struct aa_profile *profile,
                                struct aa_ruleset *rules,
                                struct aa_label *label,
                                aa_state_t state, bool subns, u32 request,
                                struct aa_perms *perms)
{
        struct aa_profile *tp;
        struct label_it i;

        /* find first subcomponent that is visible */
        label_for_each(i, label, tp) {
                if (!aa_ns_visible(profile->ns, tp->ns, subns))
                        continue;
                state = match_component(profile, rules, tp, state);
                if (!state)
                        goto fail;
                goto next;
        }

        /* no component visible */
        *perms = allperms;
        return 0;

next:
        label_for_each_cont(i, label, tp) {
                if (!aa_ns_visible(profile->ns, tp->ns, subns))
                        continue;
                state = aa_dfa_match(rules->policy->dfa, state, "//&");
                state = match_component(profile, rules, tp, state);
                if (!state)
                        goto fail;
        }
        *perms = *aa_lookup_perms(rules->policy, state);
        aa_apply_modes_to_perms(profile, perms);
        if ((perms->allow & request) != request)
                return -EACCES;

        return 0;

fail:
        *perms = nullperms;
        return state;
}

/**
 * label_components_match - find perms for all subcomponents of a label
 * @profile: profile to find perms for
 * @rules: ruleset to search
 * @label: label to check access permissions for
 * @start: state to start match in
 * @subns: whether to do permission checks on components in a subns
 * @request: permissions to request
 * @perms: an initialized perms struct to add accumulation to
 *
 * Returns: 0 on success else ERROR
 *
 * For the label A//&B//&C this does the perm match for each of A and B and C
 * @perms should be preinitialized with allperms OR a previous permission
 *        check to be stacked.
 */
static int label_components_match(struct aa_profile *profile,
                                  struct aa_ruleset *rules,
                                  struct aa_label *label, aa_state_t start,
                                  bool subns, u32 request,
                                  struct aa_perms *perms)
{
        struct aa_profile *tp;
        struct label_it i;
        struct aa_perms tmp;
        aa_state_t state = 0;

        /* find first subcomponent to test */
        label_for_each(i, label, tp) {
                if (!aa_ns_visible(profile->ns, tp->ns, subns))
                        continue;
                state = match_component(profile, rules, tp, start);
                if (!state)
                        goto fail;
                goto next;
        }

        /* no subcomponents visible - no change in perms */
        return 0;

next:
        tmp = *aa_lookup_perms(rules->policy, state);
        aa_apply_modes_to_perms(profile, &tmp);
        aa_perms_accum(perms, &tmp);
        label_for_each_cont(i, label, tp) {
                if (!aa_ns_visible(profile->ns, tp->ns, subns))
                        continue;
                state = match_component(profile, rules, tp, start);
                if (!state)
                        goto fail;
                tmp = *aa_lookup_perms(rules->policy, state);
                aa_apply_modes_to_perms(profile, &tmp);
                aa_perms_accum(perms, &tmp);
        }

        if ((perms->allow & request) != request)
                return -EACCES;

        return 0;

fail:
        *perms = nullperms;
        return -EACCES;
}

/**
 * aa_label_match - do a multi-component label match
 * @profile: profile to match against (NOT NULL)
 * @rules: ruleset to search
 * @label: label to match (NOT NULL)
 * @state: state to start in
 * @subns: whether to match subns components
 * @request: permission request
 * @perms: Returns computed perms (NOT NULL)
 *
 * Returns: the state the match finished in, may be the none matching state
 */
int aa_label_match(struct aa_profile *profile, struct aa_ruleset *rules,
                   struct aa_label *label, aa_state_t state, bool subns,
                   u32 request, struct aa_perms *perms)
{
        int error = label_compound_match(profile, rules, label, state, subns,
                                         request, perms);
        if (!error)
                return error;

        *perms = allperms;
        return label_components_match(profile, rules, label, state, subns,
                                      request, perms);
}


/**
 * aa_update_label_name - update a label to have a stored name
 * @ns: ns being viewed from (NOT NULL)
 * @label: label to update (NOT NULL)
 * @gfp: type of memory allocation
 *
 * Requires: labels_set(label) not locked in caller
 *
 * note: only updates the label name if it does not have a name already
 *       and if it is in the labelset
 */
bool aa_update_label_name(struct aa_ns *ns, struct aa_label *label, gfp_t gfp)
{
        struct aa_labelset *ls;
        unsigned long flags;
        char __counted *name;
        bool res = false;

        AA_BUG(!ns);
        AA_BUG(!label);

        if (label->hname || labels_ns(label) != ns)
                return res;

        if (aa_label_acntsxprint(&name, ns, label, FLAGS_NONE, gfp) < 0)
                return res;

        ls = labels_set(label);
        write_lock_irqsave(&ls->lock, flags);
        if (!label->hname && label->flags & FLAG_IN_TREE) {
                label->hname = name;
                res = true;
        } else
                aa_put_str(name);
        write_unlock_irqrestore(&ls->lock, flags);

        return res;
}

/*
 * cached label name is present and visible
 * @label->hname only exists if label is namespace hierarchical
 */
static inline bool use_label_hname(struct aa_ns *ns, struct aa_label *label,
                                   int flags)
{
        if (label->hname && (!ns || labels_ns(label) == ns) &&
            !(flags & ~FLAG_SHOW_MODE))
                return true;

        return false;
}

/* helper macro for snprint routines */
#define update_for_len(total, len, size, str)        \
do {                                        \
        size_t ulen = len;                \
                                        \
        AA_BUG(len < 0);                \
        total += ulen;                        \
        ulen = min(ulen, size);                \
        size -= ulen;                        \
        str += ulen;                        \
} while (0)

/**
 * aa_profile_snxprint - print a profile name to a buffer
 * @str: buffer to write to. (MAY BE NULL if @size == 0)
 * @size: size of buffer
 * @view: namespace profile is being viewed from
 * @profile: profile to view (NOT NULL)
 * @flags: whether to include the mode string
 * @prev_ns: last ns printed when used in compound print
 *
 * Returns: size of name written or would be written if larger than
 *          available buffer
 *
 * Note: will not print anything if the profile is not visible
 */
static int aa_profile_snxprint(char *str, size_t size, struct aa_ns *view,
                               struct aa_profile *profile, int flags,
                               struct aa_ns **prev_ns)
{
        const char *ns_name = NULL;

        AA_BUG(!str && size != 0);
        AA_BUG(!profile);

        if (!view)
                view = profiles_ns(profile);

        if (view != profile->ns &&
            (!prev_ns || (*prev_ns != profile->ns))) {
                if (prev_ns)
                        *prev_ns = profile->ns;
                ns_name = aa_ns_name(view, profile->ns,
                                     flags & FLAG_VIEW_SUBNS);
                if (ns_name == aa_hidden_ns_name) {
                        if (flags & FLAG_HIDDEN_UNCONFINED)
                                return snprintf(str, size, "%s", "unconfined");
                        return snprintf(str, size, "%s", ns_name);
                }
        }

        if ((flags & FLAG_SHOW_MODE) && profile != profile->ns->unconfined) {
                const char *modestr = aa_profile_mode_names[profile->mode];

                if (ns_name)
                        return snprintf(str, size, ":%s:%s (%s)", ns_name,
                                        profile->base.hname, modestr);
                return snprintf(str, size, "%s (%s)", profile->base.hname,
                                modestr);
        }

        if (ns_name)
                return snprintf(str, size, ":%s:%s", ns_name,
                                profile->base.hname);
        return snprintf(str, size, "%s", profile->base.hname);
}

static const char *label_modename(struct aa_ns *ns, struct aa_label *label,
                                  int flags)
{
        struct aa_profile *profile;
        struct label_it i;
        int mode = -1, count = 0;

        label_for_each(i, label, profile) {
                if (aa_ns_visible(ns, profile->ns, flags & FLAG_VIEW_SUBNS)) {
                        count++;
                        if (profile == profile->ns->unconfined)
                                /* special case unconfined so stacks with
                                 * unconfined don't report as mixed. ie.
                                 * profile_foo//&:ns1:unconfined (mixed)
                                 */
                                continue;
                        if (mode == -1)
                                mode = profile->mode;
                        else if (mode != profile->mode)
                                return "mixed";
                }
        }

        if (count == 0)
                return "-";
        if (mode == -1)
                /* everything was unconfined */
                mode = APPARMOR_UNCONFINED;

        return aa_profile_mode_names[mode];
}

/* if any visible label is not unconfined the display_mode returns true */
static inline bool display_mode(struct aa_ns *ns, struct aa_label *label,
                                int flags)
{
        if ((flags & FLAG_SHOW_MODE)) {
                struct aa_profile *profile;
                struct label_it i;

                label_for_each(i, label, profile) {
                        if (aa_ns_visible(ns, profile->ns,
                                          flags & FLAG_VIEW_SUBNS) &&
                            profile != profile->ns->unconfined)
                                return true;
                }
                /* only ns->unconfined in set of profiles in ns */
                return false;
        }

        return false;
}

/**
 * aa_label_snxprint - print a label name to a string buffer
 * @str: buffer to write to. (MAY BE NULL if @size == 0)
 * @size: size of buffer
 * @ns: namespace profile is being viewed from
 * @label: label to view (NOT NULL)
 * @flags: whether to include the mode string
 *
 * Returns: size of name written or would be written if larger than
 *          available buffer
 *
 * Note: labels do not have to be strictly hierarchical to the ns as
 *       objects may be shared across different namespaces and thus
 *       pickup labeling from each ns.  If a particular part of the
 *       label is not visible it will just be excluded.  And if none
 *       of the label is visible "---" will be used.
 */
int aa_label_snxprint(char *str, size_t size, struct aa_ns *ns,
                      struct aa_label *label, int flags)
{
        struct aa_profile *profile;
        struct aa_ns *prev_ns = NULL;
        struct label_it i;
        int count = 0, total = 0;
        ssize_t len;

        AA_BUG(!str && size != 0);
        AA_BUG(!label);

        if (DEBUG_ABS_ROOT && (flags & FLAG_ABS_ROOT)) {
                ns = root_ns;
                len = snprintf(str, size, "_");
                update_for_len(total, len, size, str);
        } else if (!ns) {
                ns = labels_ns(label);
        }

        label_for_each(i, label, profile) {
                if (aa_ns_visible(ns, profile->ns, flags & FLAG_VIEW_SUBNS)) {
                        if (count > 0) {
                                len = snprintf(str, size, "//&");
                                update_for_len(total, len, size, str);
                        }
                        len = aa_profile_snxprint(str, size, ns, profile,
                                                  flags & FLAG_VIEW_SUBNS,
                                                  &prev_ns);
                        update_for_len(total, len, size, str);
                        count++;
                }
        }

        if (count == 0) {
                if (flags & FLAG_HIDDEN_UNCONFINED)
                        return snprintf(str, size, "%s", "unconfined");
                return snprintf(str, size, "%s", aa_hidden_ns_name);
        }

        /* count == 1 && ... is for backwards compat where the mode
         * is not displayed for 'unconfined' in the current ns
         */
        if (display_mode(ns, label, flags)) {
                len = snprintf(str, size, " (%s)",
                               label_modename(ns, label, flags));
                update_for_len(total, len, size, str);
        }

        return total;
}
#undef update_for_len

/**
 * aa_label_asxprint - allocate a string buffer and print label into it
 * @strp: Returns - the allocated buffer with the label name. (NOT NULL)
 * @ns: namespace profile is being viewed from
 * @label: label to view (NOT NULL)
 * @flags: flags controlling what label info is printed
 * @gfp: kernel memory allocation type
 *
 * Returns: size of name written or would be written if larger than
 *          available buffer
 */
int aa_label_asxprint(char **strp, struct aa_ns *ns, struct aa_label *label,
                      int flags, gfp_t gfp)
{
        int size;

        AA_BUG(!strp);
        AA_BUG(!label);

        size = aa_label_snxprint(NULL, 0, ns, label, flags);
        if (size < 0)
                return size;

        *strp = kmalloc(size + 1, gfp);
        if (!*strp)
                return -ENOMEM;
        return aa_label_snxprint(*strp, size + 1, ns, label, flags);
}

/**
 * aa_label_acntsxprint - allocate a __counted string buffer and print label
 * @strp: buffer to write to.
 * @ns: namespace profile is being viewed from
 * @label: label to view (NOT NULL)
 * @flags: flags controlling what label info is printed
 * @gfp: kernel memory allocation type
 *
 * Returns: size of name written or would be written if larger than
 *          available buffer
 */
int aa_label_acntsxprint(char __counted **strp, struct aa_ns *ns,
                         struct aa_label *label, int flags, gfp_t gfp)
{
        int size;

        AA_BUG(!strp);
        AA_BUG(!label);

        size = aa_label_snxprint(NULL, 0, ns, label, flags);
        if (size < 0)
                return size;

        *strp = aa_str_alloc(size + 1, gfp);
        if (!*strp)
                return -ENOMEM;
        return aa_label_snxprint(*strp, size + 1, ns, label, flags);
}


void aa_label_xaudit(struct audit_buffer *ab, struct aa_ns *ns,
                     struct aa_label *label, int flags, gfp_t gfp)
{
        const char *str;
        char *name = NULL;
        int len;

        AA_BUG(!ab);
        AA_BUG(!label);

        if (!use_label_hname(ns, label, flags) ||
            display_mode(ns, label, flags)) {
                len  = aa_label_asxprint(&name, ns, label, flags, gfp);
                if (len < 0) {
                        AA_DEBUG(DEBUG_LABEL, "label print error");
                        return;
                }
                str = name;
        } else {
                str = (char *) label->hname;
                len = strlen(str);
        }
        if (audit_string_contains_control(str, len))
                audit_log_n_hex(ab, str, len);
        else
                audit_log_n_string(ab, str, len);

        kfree(name);
}

void aa_label_seq_xprint(struct seq_file *f, struct aa_ns *ns,
                         struct aa_label *label, int flags, gfp_t gfp)
{
        AA_BUG(!f);
        AA_BUG(!label);

        if (!use_label_hname(ns, label, flags)) {
                char *str;
                int len;

                len = aa_label_asxprint(&str, ns, label, flags, gfp);
                if (len < 0) {
                        AA_DEBUG(DEBUG_LABEL, "label print error");
                        return;
                }
                seq_puts(f, str);
                kfree(str);
        } else if (display_mode(ns, label, flags))
                seq_printf(f, "%s (%s)", label->hname,
                           label_modename(ns, label, flags));
        else
                seq_puts(f, label->hname);
}

void aa_label_xprintk(struct aa_ns *ns, struct aa_label *label, int flags,
                      gfp_t gfp)
{
        AA_BUG(!label);

        if (!use_label_hname(ns, label, flags)) {
                char *str;
                int len;

                len = aa_label_asxprint(&str, ns, label, flags, gfp);
                if (len < 0) {
                        AA_DEBUG(DEBUG_LABEL, "label print error");
                        return;
                }
                pr_info("%s", str);
                kfree(str);
        } else if (display_mode(ns, label, flags))
                pr_info("%s (%s)", label->hname,
                       label_modename(ns, label, flags));
        else
                pr_info("%s", label->hname);
}

void aa_label_printk(struct aa_label *label, gfp_t gfp)
{
        struct aa_ns *ns = aa_get_current_ns();

        aa_label_xprintk(ns, label, FLAG_VIEW_SUBNS, gfp);
        aa_put_ns(ns);
}

static int label_count_strn_entries(const char *str, size_t n)
{
        const char *end = str + n;
        const char *split;
        int count = 1;

        AA_BUG(!str);

        for (split = aa_label_strn_split(str, end - str);
             split;
             split = aa_label_strn_split(str, end - str)) {
                count++;
                str = split + 3;
        }

        return count;
}

/*
 * ensure stacks with components like
 *   :ns:A//&B
 * have :ns: applied to both 'A' and 'B' by making the lookup relative
 * to the base if the lookup specifies an ns, else making the stacked lookup
 * relative to the last embedded ns in the string.
 */
static struct aa_profile *fqlookupn_profile(struct aa_label *base,
                                            struct aa_label *currentbase,
                                            const char *str, size_t n)
{
        const char *first = skipn_spaces(str, n);

        if (first && *first == ':')
                return aa_fqlookupn_profile(base, str, n);

        return aa_fqlookupn_profile(currentbase, str, n);
}

/**
 * aa_label_strn_parse - parse, validate and convert a text string to a label
 * @base: base label to use for lookups (NOT NULL)
 * @str: null terminated text string (NOT NULL)
 * @n: length of str to parse, will stop at \0 if encountered before n
 * @gfp: allocation type
 * @create: true if should create compound labels if they don't exist
 * @force_stack: true if should stack even if no leading &
 *
 * Returns: the matching refcounted label if present
 *     else ERRPTR
 */
struct aa_label *aa_label_strn_parse(struct aa_label *base, const char *str,
                                     size_t n, gfp_t gfp, bool create,
                                     bool force_stack)
{
        DEFINE_VEC(profile, vec);
        struct aa_label *label, *currbase = base;
        int i, len, stack = 0, error;
        const char *end = str + n;
        const char *split;

        AA_BUG(!base);
        AA_BUG(!str);

        str = skipn_spaces(str, n);
        if (str == NULL || (DEBUG_ABS_ROOT && *str == '_' &&
                            base != &root_ns->unconfined->label))
                return ERR_PTR(-EINVAL);

        len = label_count_strn_entries(str, end - str);
        if (*str == '&' || force_stack) {
                /* stack on top of base */
                stack = base->size;
                len += stack;
                if (*str == '&')
                        str++;
        }

        error = vec_setup(profile, vec, len, gfp);
        if (error)
                return ERR_PTR(error);

        for (i = 0; i < stack; i++)
                vec[i] = aa_get_profile(base->vec[i]);

        for (split = aa_label_strn_split(str, end - str), i = stack;
             split && i < len; i++) {
                vec[i] = fqlookupn_profile(base, currbase, str, split - str);
                if (!vec[i])
                        goto fail;
                /*
                 * if component specified a new ns it becomes the new base
                 * so that subsequent lookups are relative to it
                 */
                if (vec[i]->ns != labels_ns(currbase))
                        currbase = &vec[i]->label;
                str = split + 3;
                split = aa_label_strn_split(str, end - str);
        }
        /* last element doesn't have a split */
        if (i < len) {
                vec[i] = fqlookupn_profile(base, currbase, str, end - str);
                if (!vec[i])
                        goto fail;
        }
        if (len == 1)
                /* no need to free vec as len < LOCAL_VEC_ENTRIES */
                return &vec[0]->label;

        len -= aa_vec_unique(vec, len, VEC_FLAG_TERMINATE);
        /* TODO: deal with reference labels */
        if (len == 1) {
                label = aa_get_label(&vec[0]->label);
                goto out;
        }

        if (create)
                label = aa_vec_find_or_create_label(vec, len, gfp);
        else
                label = vec_find(vec, len);
        if (!label)
                goto fail;

out:
        /* use adjusted len from after vec_unique, not original */
        vec_cleanup(profile, vec, len);
        return label;

fail:
        label = ERR_PTR(-ENOENT);
        goto out;
}

struct aa_label *aa_label_parse(struct aa_label *base, const char *str,
                                gfp_t gfp, bool create, bool force_stack)
{
        return aa_label_strn_parse(base, str, strlen(str), gfp, create,
                                   force_stack);
}

/**
 * aa_labelset_destroy - remove all labels from the label set
 * @ls: label set to cleanup (NOT NULL)
 *
 * Labels that are removed from the set may still exist beyond the set
 * being destroyed depending on their reference counting
 */
void aa_labelset_destroy(struct aa_labelset *ls)
{
        struct rb_node *node;
        unsigned long flags;

        AA_BUG(!ls);

        write_lock_irqsave(&ls->lock, flags);
        for (node = rb_first(&ls->root); node; node = rb_first(&ls->root)) {
                struct aa_label *this = rb_entry(node, struct aa_label, node);

                if (labels_ns(this) != root_ns)
                        __label_remove(this,
                                       ns_unconfined(labels_ns(this)->parent));
                else
                        __label_remove(this, NULL);
        }
        write_unlock_irqrestore(&ls->lock, flags);
}

/*
 * @ls: labelset to init (NOT NULL)
 */
void aa_labelset_init(struct aa_labelset *ls)
{
        AA_BUG(!ls);

        rwlock_init(&ls->lock);
        ls->root = RB_ROOT;
}

static struct aa_label *labelset_next_stale(struct aa_labelset *ls)
{
        struct aa_label *label;
        struct rb_node *node;
        unsigned long flags;

        AA_BUG(!ls);

        read_lock_irqsave(&ls->lock, flags);

        __labelset_for_each(ls, node) {
                label = rb_entry(node, struct aa_label, node);
                if ((label_is_stale(label) ||
                     vec_is_stale(label->vec, label->size)) &&
                    __aa_get_label(label))
                        goto out;

        }
        label = NULL;

out:
        read_unlock_irqrestore(&ls->lock, flags);

        return label;
}

/**
 * __label_update - insert updated version of @label into labelset
 * @label: the label to update/replace
 *
 * Returns: new label that is up to date
 *     else NULL on failure
 *
 * Requires: @ns lock be held
 *
 * Note: worst case is the stale @label does not get updated and has
 *       to be updated at a later time.
 */
static struct aa_label *__label_update(struct aa_label *label)
{
        struct aa_label *new, *tmp;
        struct aa_labelset *ls;
        unsigned long flags;
        int i, invcount = 0;

        AA_BUG(!label);
        AA_BUG(!mutex_is_locked(&labels_ns(label)->lock));

        new = aa_label_alloc(label->size, label->proxy, GFP_KERNEL);
        if (!new)
                return NULL;

        /*
         * while holding the ns_lock will stop profile replacement, removal,
         * and label updates, label merging and removal can be occurring
         */
        ls = labels_set(label);
        write_lock_irqsave(&ls->lock, flags);
        for (i = 0; i < label->size; i++) {
                AA_BUG(!label->vec[i]);
                new->vec[i] = aa_get_newest_profile(label->vec[i]);
                AA_BUG(!new->vec[i]);
                AA_BUG(!new->vec[i]->label.proxy);
                AA_BUG(!new->vec[i]->label.proxy->label);
                if (new->vec[i]->label.proxy != label->vec[i]->label.proxy)
                        invcount++;
        }

        /* updated stale label by being removed/renamed from labelset */
        if (invcount) {
                new->size -= aa_vec_unique(&new->vec[0], new->size,
                                           VEC_FLAG_TERMINATE);
                /* TODO: deal with reference labels */
                if (new->size == 1) {
                        tmp = aa_get_label(&new->vec[0]->label);
                        AA_BUG(tmp == label);
                        goto remove;
                }
                if (labels_set(label) != labels_set(new)) {
                        write_unlock_irqrestore(&ls->lock, flags);
                        tmp = aa_label_insert(labels_set(new), new);
                        write_lock_irqsave(&ls->lock, flags);
                        goto remove;
                }
        } else
                AA_BUG(labels_ns(label) != labels_ns(new));

        tmp = __label_insert(labels_set(label), new, true);
remove:
        /* ensure label is removed, and redirected correctly */
        __label_remove(label, tmp);
        write_unlock_irqrestore(&ls->lock, flags);
        label_free_or_put_new(tmp, new);

        return tmp;
}

/**
 * __labelset_update - update labels in @ns
 * @ns: namespace to update labels in  (NOT NULL)
 *
 * Requires: @ns lock be held
 *
 * Walk the labelset ensuring that all labels are up to date and valid
 * Any label that has a stale component is marked stale and replaced and
 * by an updated version.
 *
 * If failures happen due to memory pressures then stale labels will
 * be left in place until the next pass.
 */
static void __labelset_update(struct aa_ns *ns)
{
        struct aa_label *label;

        AA_BUG(!ns);
        AA_BUG(!mutex_is_locked(&ns->lock));

        do {
                label = labelset_next_stale(&ns->labels);
                if (label) {
                        struct aa_label *l = __label_update(label);

                        aa_put_label(l);
                        aa_put_label(label);
                }
        } while (label);
}

/**
 * __aa_labelset_update_subtree - update all labels with a stale component
 * @ns: ns to start update at (NOT NULL)
 *
 * Requires: @ns lock be held
 *
 * Invalidates labels based on @p in @ns and any children namespaces.
 */
void __aa_labelset_update_subtree(struct aa_ns *ns)
{
        struct aa_ns *child;

        AA_BUG(!ns);
        AA_BUG(!mutex_is_locked(&ns->lock));

        __labelset_update(ns);

        list_for_each_entry(child, &ns->sub_ns, base.list) {
                mutex_lock_nested(&child->lock, child->level);
                __aa_labelset_update_subtree(child);
                mutex_unlock(&child->lock);
        }
}



















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _VT_KERN_H
#define _VT_KERN_H

/*
 * this really is an extension of the vc_cons structure in console.c, but
 * with information needed by the vt package
 */

#include <linux/vt.h>
#include <linux/kd.h>
#include <linux/tty.h>
#include <linux/mutex.h>
#include <linux/console_struct.h>
#include <linux/mm.h>
#include <linux/consolemap.h>
#include <linux/notifier.h>

void kd_mksound(unsigned int hz, unsigned int ticks);
int kbd_rate(struct kbd_repeat *rep);

extern int fg_console, last_console, want_console;

/* console.c */

int vc_allocate(unsigned int console);
int vc_cons_allocated(unsigned int console);
int __vc_resize(struct vc_data *vc, unsigned int cols, unsigned int lines,
                bool from_user);
struct vc_data *vc_deallocate(unsigned int console);
void reset_palette(struct vc_data *vc);
void do_blank_screen(int entering_gfx);
void do_unblank_screen(int leaving_gfx);
void poke_blanked_console(void);
int con_font_op(struct vc_data *vc, struct console_font_op *op);
int con_set_cmap(unsigned char __user *cmap);
int con_get_cmap(unsigned char __user *cmap);
void scrollback(struct vc_data *vc);
void scrollfront(struct vc_data *vc, int lines);
void clear_buffer_attributes(struct vc_data *vc);
void update_region(struct vc_data *vc, unsigned long start, int count);
void redraw_screen(struct vc_data *vc, int is_switch);
#define update_screen(x) redraw_screen(x, 0)
#define switch_screen(x) redraw_screen(x, 1)

static inline int vc_resize(struct vc_data *vc, unsigned int cols,
                            unsigned int lines)
{
        return __vc_resize(vc, cols, lines, false);
}

struct tty_struct;
int tioclinux(struct tty_struct *tty, unsigned long arg);

#ifdef CONFIG_CONSOLE_TRANSLATIONS
/* consolemap.c */

struct unipair;

int con_set_trans_old(unsigned char __user * table);
int con_get_trans_old(unsigned char __user * table);
int con_set_trans_new(unsigned short __user * table);
int con_get_trans_new(unsigned short __user * table);
int con_clear_unimap(struct vc_data *vc);
int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list);
int con_get_unimap(struct vc_data *vc, ushort ct, ushort __user *uct, struct unipair __user *list);
int con_set_default_unimap(struct vc_data *vc);
void con_free_unimap(struct vc_data *vc);
int con_copy_unimap(struct vc_data *dst_vc, struct vc_data *src_vc);

#else
static inline int con_set_trans_old(unsigned char __user *table)
{
        return 0;
}
static inline int con_get_trans_old(unsigned char __user *table)
{
        return -EINVAL;
}
static inline int con_set_trans_new(unsigned short __user *table)
{
        return 0;
}
static inline int con_get_trans_new(unsigned short __user *table)
{
        return -EINVAL;
}
static inline int con_clear_unimap(struct vc_data *vc)
{
        return 0;
}
static inline
int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list)
{
        return 0;
}
static inline
int con_get_unimap(struct vc_data *vc, ushort ct, ushort __user *uct,
                   struct unipair __user *list)
{
        return -EINVAL;
}
static inline int con_set_default_unimap(struct vc_data *vc)
{
        return 0;
}
static inline void con_free_unimap(struct vc_data *vc)
{
}
static inline void con_protect_unimap(struct vc_data *vc, int rdonly)
{
}
static inline
int con_copy_unimap(struct vc_data *dst_vc, struct vc_data *src_vc)
{
        return 0;
}

#endif

/* vt.c */
void vt_event_post(unsigned int event, unsigned int old, unsigned int new);
int vt_waitactive(int n);
void change_console(struct vc_data *new_vc);
void reset_vc(struct vc_data *vc);
int do_unbind_con_driver(const struct consw *csw, int first, int last,
                         int deflt);
int vty_init(const struct file_operations *console_fops);

extern bool vt_dont_switch;
extern int default_utf8;
extern int global_cursor_default;

struct vt_spawn_console {
        spinlock_t lock;
        struct pid *pid;
        int sig;
};
extern struct vt_spawn_console vt_spawn_con;

int vt_move_to_console(unsigned int vt, int alloc);

/* Interfaces for VC notification of character events (for accessibility etc) */

struct vt_notifier_param {
        struct vc_data *vc;        /* VC on which the update happened */
        unsigned int c;                /* Printed char */
};

int register_vt_notifier(struct notifier_block *nb);
int unregister_vt_notifier(struct notifier_block *nb);

void hide_boot_cursor(bool hide);

/* keyboard  provided interfaces */
int vt_do_diacrit(unsigned int cmd, void __user *up, int eperm);
int vt_do_kdskbmode(unsigned int console, unsigned int arg);
int vt_do_kdskbmeta(unsigned int console, unsigned int arg);
int vt_do_kbkeycode_ioctl(int cmd, struct kbkeycode __user *user_kbkc,
                          int perm);
int vt_do_kdsk_ioctl(int cmd, struct kbentry __user *user_kbe, int perm,
                     unsigned int console);
int vt_do_kdgkb_ioctl(int cmd, struct kbsentry __user *user_kdgkb, int perm);
int vt_do_kdskled(unsigned int console, int cmd, unsigned long arg, int perm);
int vt_do_kdgkbmode(unsigned int console);
int vt_do_kdgkbmeta(unsigned int console);
void vt_reset_unicode(unsigned int console);
int vt_get_shift_state(void);
void vt_reset_keyboard(unsigned int console);
int vt_get_leds(unsigned int console, int flag);
int vt_get_kbd_mode_bit(unsigned int console, int bit);
void vt_set_kbd_mode_bit(unsigned int console, int bit);
void vt_clr_kbd_mode_bit(unsigned int console, int bit);
void vt_set_led_state(unsigned int console, int leds);
void vt_kbd_con_start(unsigned int console);
void vt_kbd_con_stop(unsigned int console);

#endif /* _VT_KERN_H */









































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    1 












    1 



































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945

/*
 * Copyright (c) 2011 Atheros Communications Inc.
 * Copyright (c) 2011-2012 Qualcomm Atheros, Inc.
 *
 * Permission to use, copy, modify, and/or distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/moduleparam.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/of.h>
#include <linux/mmc/sdio_func.h>
#include <linux/vmalloc.h>

#include "core.h"
#include "cfg80211.h"
#include "target.h"
#include "debug.h"
#include "hif-ops.h"
#include "htc-ops.h"

static const struct ath6kl_hw hw_list[] = {
        {
                .id                                = AR6003_HW_2_0_VERSION,
                .name                                = "ar6003 hw 2.0",
                .dataset_patch_addr                = 0x57e884,
                .app_load_addr                        = 0x543180,
                .board_ext_data_addr                = 0x57e500,
                .reserved_ram_size                = 6912,
                .refclk_hz                        = 26000000,
                .uarttx_pin                        = 8,
                .flags                                = ATH6KL_HW_SDIO_CRC_ERROR_WAR,

                /* hw2.0 needs override address hardcoded */
                .app_start_override_addr        = 0x944C00,

                .fw = {
                        .dir                = AR6003_HW_2_0_FW_DIR,
                        .otp                = AR6003_HW_2_0_OTP_FILE,
                        .fw                = AR6003_HW_2_0_FIRMWARE_FILE,
                        .tcmd                = AR6003_HW_2_0_TCMD_FIRMWARE_FILE,
                        .patch                = AR6003_HW_2_0_PATCH_FILE,
                },

                .fw_board                = AR6003_HW_2_0_BOARD_DATA_FILE,
                .fw_default_board        = AR6003_HW_2_0_DEFAULT_BOARD_DATA_FILE,
        },
        {
                .id                                = AR6003_HW_2_1_1_VERSION,
                .name                                = "ar6003 hw 2.1.1",
                .dataset_patch_addr                = 0x57ff74,
                .app_load_addr                        = 0x1234,
                .board_ext_data_addr                = 0x542330,
                .reserved_ram_size                = 512,
                .refclk_hz                        = 26000000,
                .uarttx_pin                        = 8,
                .testscript_addr                = 0x57ef74,
                .flags                                = ATH6KL_HW_SDIO_CRC_ERROR_WAR,

                .fw = {
                        .dir                = AR6003_HW_2_1_1_FW_DIR,
                        .otp                = AR6003_HW_2_1_1_OTP_FILE,
                        .fw                = AR6003_HW_2_1_1_FIRMWARE_FILE,
                        .tcmd                = AR6003_HW_2_1_1_TCMD_FIRMWARE_FILE,
                        .patch                = AR6003_HW_2_1_1_PATCH_FILE,
                        .utf                = AR6003_HW_2_1_1_UTF_FIRMWARE_FILE,
                        .testscript        = AR6003_HW_2_1_1_TESTSCRIPT_FILE,
                },

                .fw_board                = AR6003_HW_2_1_1_BOARD_DATA_FILE,
                .fw_default_board = AR6003_HW_2_1_1_DEFAULT_BOARD_DATA_FILE,
        },
        {
                .id                                = AR6004_HW_1_0_VERSION,
                .name                                = "ar6004 hw 1.0",
                .dataset_patch_addr                = 0x57e884,
                .app_load_addr                        = 0x1234,
                .board_ext_data_addr                = 0x437000,
                .reserved_ram_size                = 19456,
                .board_addr                        = 0x433900,
                .refclk_hz                        = 26000000,
                .uarttx_pin                        = 11,
                .flags                                = 0,

                .fw = {
                        .dir                = AR6004_HW_1_0_FW_DIR,
                        .fw                = AR6004_HW_1_0_FIRMWARE_FILE,
                },

                .fw_board                = AR6004_HW_1_0_BOARD_DATA_FILE,
                .fw_default_board        = AR6004_HW_1_0_DEFAULT_BOARD_DATA_FILE,
        },
        {
                .id                                = AR6004_HW_1_1_VERSION,
                .name                                = "ar6004 hw 1.1",
                .dataset_patch_addr                = 0x57e884,
                .app_load_addr                        = 0x1234,
                .board_ext_data_addr                = 0x437000,
                .reserved_ram_size                = 11264,
                .board_addr                        = 0x43d400,
                .refclk_hz                        = 40000000,
                .uarttx_pin                        = 11,
                .flags                                = 0,
                .fw = {
                        .dir                = AR6004_HW_1_1_FW_DIR,
                        .fw                = AR6004_HW_1_1_FIRMWARE_FILE,
                },

                .fw_board                = AR6004_HW_1_1_BOARD_DATA_FILE,
                .fw_default_board        = AR6004_HW_1_1_DEFAULT_BOARD_DATA_FILE,
        },
        {
                .id                                = AR6004_HW_1_2_VERSION,
                .name                                = "ar6004 hw 1.2",
                .dataset_patch_addr                = 0x436ecc,
                .app_load_addr                        = 0x1234,
                .board_ext_data_addr                = 0x437000,
                .reserved_ram_size                = 9216,
                .board_addr                        = 0x435c00,
                .refclk_hz                        = 40000000,
                .uarttx_pin                        = 11,
                .flags                                = 0,

                .fw = {
                        .dir                = AR6004_HW_1_2_FW_DIR,
                        .fw                = AR6004_HW_1_2_FIRMWARE_FILE,
                },
                .fw_board                = AR6004_HW_1_2_BOARD_DATA_FILE,
                .fw_default_board        = AR6004_HW_1_2_DEFAULT_BOARD_DATA_FILE,
        },
        {
                .id                                = AR6004_HW_1_3_VERSION,
                .name                                = "ar6004 hw 1.3",
                .dataset_patch_addr                = 0x437860,
                .app_load_addr                        = 0x1234,
                .board_ext_data_addr                = 0x437000,
                .reserved_ram_size                = 7168,
                .board_addr                        = 0x436400,
                .refclk_hz                      = 0,
                .uarttx_pin                     = 11,
                .flags                                = 0,

                .fw = {
                        .dir            = AR6004_HW_1_3_FW_DIR,
                        .fw             = AR6004_HW_1_3_FIRMWARE_FILE,
                        .tcmd                = AR6004_HW_1_3_TCMD_FIRMWARE_FILE,
                        .utf                = AR6004_HW_1_3_UTF_FIRMWARE_FILE,
                        .testscript        = AR6004_HW_1_3_TESTSCRIPT_FILE,
                },

                .fw_board               = AR6004_HW_1_3_BOARD_DATA_FILE,
                .fw_default_board       = AR6004_HW_1_3_DEFAULT_BOARD_DATA_FILE,
        },
        {
                .id                                = AR6004_HW_3_0_VERSION,
                .name                                = "ar6004 hw 3.0",
                .dataset_patch_addr                = 0,
                .app_load_addr                        = 0x1234,
                .board_ext_data_addr                = 0,
                .reserved_ram_size                = 7168,
                .board_addr                        = 0x436400,
                .testscript_addr                = 0,
                .uarttx_pin                        = 11,
                .flags                                = 0,

                .fw = {
                        .dir                = AR6004_HW_3_0_FW_DIR,
                        .fw                = AR6004_HW_3_0_FIRMWARE_FILE,
                        .tcmd                = AR6004_HW_3_0_TCMD_FIRMWARE_FILE,
                        .utf                = AR6004_HW_3_0_UTF_FIRMWARE_FILE,
                        .testscript        = AR6004_HW_3_0_TESTSCRIPT_FILE,
                },

                .fw_board                = AR6004_HW_3_0_BOARD_DATA_FILE,
                .fw_default_board        = AR6004_HW_3_0_DEFAULT_BOARD_DATA_FILE,
        },
};

/*
 * Include definitions here that can be used to tune the WLAN module
 * behavior. Different customers can tune the behavior as per their needs,
 * here.
 */

/*
 * This configuration item enable/disable keepalive support.
 * Keepalive support: In the absence of any data traffic to AP, null
 * frames will be sent to the AP at periodic interval, to keep the association
 * active. This configuration item defines the periodic interval.
 * Use value of zero to disable keepalive support
 * Default: 60 seconds
 */
#define WLAN_CONFIG_KEEP_ALIVE_INTERVAL 60

/*
 * This configuration item sets the value of disconnect timeout
 * Firmware delays sending the disconnect event to the host for this
 * timeout after is gets disconnected from the current AP.
 * If the firmware successly roams within the disconnect timeout
 * it sends a new connect event
 */
#define WLAN_CONFIG_DISCONNECT_TIMEOUT 10


#define ATH6KL_DATA_OFFSET    64
struct sk_buff *ath6kl_buf_alloc(int size)
{
        struct sk_buff *skb;
        u16 reserved;

        /* Add cacheline space at front and back of buffer */
        reserved = roundup((2 * L1_CACHE_BYTES) + ATH6KL_DATA_OFFSET +
                   sizeof(struct htc_packet) + ATH6KL_HTC_ALIGN_BYTES, 4);
        skb = dev_alloc_skb(size + reserved);

        if (skb)
                skb_reserve(skb, reserved - L1_CACHE_BYTES);
        return skb;
}

void ath6kl_init_profile_info(struct ath6kl_vif *vif)
{
        vif->ssid_len = 0;
        memset(vif->ssid, 0, sizeof(vif->ssid));

        vif->dot11_auth_mode = OPEN_AUTH;
        vif->auth_mode = NONE_AUTH;
        vif->prwise_crypto = NONE_CRYPT;
        vif->prwise_crypto_len = 0;
        vif->grp_crypto = NONE_CRYPT;
        vif->grp_crypto_len = 0;
        memset(vif->wep_key_list, 0, sizeof(vif->wep_key_list));
        memset(vif->req_bssid, 0, sizeof(vif->req_bssid));
        memset(vif->bssid, 0, sizeof(vif->bssid));
        vif->bss_ch = 0;
}

static int ath6kl_set_host_app_area(struct ath6kl *ar)
{
        u32 address, data;
        struct host_app_area host_app_area;

        /* Fetch the address of the host_app_area_s
         * instance in the host interest area */
        address = ath6kl_get_hi_item_addr(ar, HI_ITEM(hi_app_host_interest));
        address = TARG_VTOP(ar->target_type, address);

        if (ath6kl_diag_read32(ar, address, &data))
                return -EIO;

        address = TARG_VTOP(ar->target_type, data);
        host_app_area.wmi_protocol_ver = cpu_to_le32(WMI_PROTOCOL_VERSION);
        if (ath6kl_diag_write(ar, address, (u8 *) &host_app_area,
                              sizeof(struct host_app_area)))
                return -EIO;

        return 0;
}

static inline void set_ac2_ep_map(struct ath6kl *ar,
                                  u8 ac,
                                  enum htc_endpoint_id ep)
{
        ar->ac2ep_map[ac] = ep;
        ar->ep2ac_map[ep] = ac;
}

/* connect to a service */
static int ath6kl_connectservice(struct ath6kl *ar,
                                 struct htc_service_connect_req  *con_req,
                                 char *desc)
{
        int status;
        struct htc_service_connect_resp response;

        memset(&response, 0, sizeof(response));

        status = ath6kl_htc_conn_service(ar->htc_target, con_req, &response);
        if (status) {
                ath6kl_err("failed to connect to %s service status:%d\n",
                           desc, status);
                return status;
        }

        switch (con_req->svc_id) {
        case WMI_CONTROL_SVC:
                if (test_bit(WMI_ENABLED, &ar->flag))
                        ath6kl_wmi_set_control_ep(ar->wmi, response.endpoint);
                ar->ctrl_ep = response.endpoint;
                break;
        case WMI_DATA_BE_SVC:
                set_ac2_ep_map(ar, WMM_AC_BE, response.endpoint);
                break;
        case WMI_DATA_BK_SVC:
                set_ac2_ep_map(ar, WMM_AC_BK, response.endpoint);
                break;
        case WMI_DATA_VI_SVC:
                set_ac2_ep_map(ar, WMM_AC_VI, response.endpoint);
                break;
        case WMI_DATA_VO_SVC:
                set_ac2_ep_map(ar, WMM_AC_VO, response.endpoint);
                break;
        default:
                ath6kl_err("service id is not mapped %d\n", con_req->svc_id);
                return -EINVAL;
        }

        return 0;
}

static int ath6kl_init_service_ep(struct ath6kl *ar)
{
        struct htc_service_connect_req connect;

        memset(&connect, 0, sizeof(connect));

        /* these fields are the same for all service endpoints */
        connect.ep_cb.tx_comp_multi = ath6kl_tx_complete;
        connect.ep_cb.rx = ath6kl_rx;
        connect.ep_cb.rx_refill = ath6kl_rx_refill;
        connect.ep_cb.tx_full = ath6kl_tx_queue_full;

        /*
         * Set the max queue depth so that our ath6kl_tx_queue_full handler
         * gets called.
        */
        connect.max_txq_depth = MAX_DEFAULT_SEND_QUEUE_DEPTH;
        connect.ep_cb.rx_refill_thresh = ATH6KL_MAX_RX_BUFFERS / 4;
        if (!connect.ep_cb.rx_refill_thresh)
                connect.ep_cb.rx_refill_thresh++;

        /* connect to control service */
        connect.svc_id = WMI_CONTROL_SVC;
        if (ath6kl_connectservice(ar, &connect, "WMI CONTROL"))
                return -EIO;

        connect.flags |= HTC_FLGS_TX_BNDL_PAD_EN;

        /*
         * Limit the HTC message size on the send path, although e can
         * receive A-MSDU frames of 4K, we will only send ethernet-sized
         * (802.3) frames on the send path.
         */
        connect.max_rxmsg_sz = WMI_MAX_TX_DATA_FRAME_LENGTH;

        /*
         * To reduce the amount of committed memory for larger A_MSDU
         * frames, use the recv-alloc threshold mechanism for larger
         * packets.
         */
        connect.ep_cb.rx_alloc_thresh = ATH6KL_BUFFER_SIZE;
        connect.ep_cb.rx_allocthresh = ath6kl_alloc_amsdu_rxbuf;

        /*
         * For the remaining data services set the connection flag to
         * reduce dribbling, if configured to do so.
         */
        connect.conn_flags |= HTC_CONN_FLGS_REDUCE_CRED_DRIB;
        connect.conn_flags &= ~HTC_CONN_FLGS_THRESH_MASK;
        connect.conn_flags |= HTC_CONN_FLGS_THRESH_LVL_HALF;

        connect.svc_id = WMI_DATA_BE_SVC;

        if (ath6kl_connectservice(ar, &connect, "WMI DATA BE"))
                return -EIO;

        /* connect to back-ground map this to WMI LOW_PRI */
        connect.svc_id = WMI_DATA_BK_SVC;
        if (ath6kl_connectservice(ar, &connect, "WMI DATA BK"))
                return -EIO;

        /* connect to Video service, map this to HI PRI */
        connect.svc_id = WMI_DATA_VI_SVC;
        if (ath6kl_connectservice(ar, &connect, "WMI DATA VI"))
                return -EIO;

        /*
         * Connect to VO service, this is currently not mapped to a WMI
         * priority stream due to historical reasons. WMI originally
         * defined 3 priorities over 3 mailboxes We can change this when
         * WMI is reworked so that priorities are not dependent on
         * mailboxes.
         */
        connect.svc_id = WMI_DATA_VO_SVC;
        if (ath6kl_connectservice(ar, &connect, "WMI DATA VO"))
                return -EIO;

        return 0;
}

void ath6kl_init_control_info(struct ath6kl_vif *vif)
{
        ath6kl_init_profile_info(vif);
        vif->def_txkey_index = 0;
        memset(vif->wep_key_list, 0, sizeof(vif->wep_key_list));
        vif->ch_hint = 0;
}

/*
 * Set HTC/Mbox operational parameters, this can only be called when the
 * target is in the BMI phase.
 */
static int ath6kl_set_htc_params(struct ath6kl *ar, u32 mbox_isr_yield_val,
                                 u8 htc_ctrl_buf)
{
        int status;
        u32 blk_size;

        blk_size = ar->mbox_info.block_size;

        if (htc_ctrl_buf)
                blk_size |=  ((u32)htc_ctrl_buf) << 16;

        /* set the host interest area for the block size */
        status = ath6kl_bmi_write_hi32(ar, hi_mbox_io_block_sz, blk_size);
        if (status) {
                ath6kl_err("bmi_write_memory for IO block size failed\n");
                goto out;
        }

        ath6kl_dbg(ATH6KL_DBG_TRC, "block size set: %d (target addr:0x%X)\n",
                   blk_size,
                   ath6kl_get_hi_item_addr(ar, HI_ITEM(hi_mbox_io_block_sz)));

        if (mbox_isr_yield_val) {
                /* set the host interest area for the mbox ISR yield limit */
                status = ath6kl_bmi_write_hi32(ar, hi_mbox_isr_yield_limit,
                                               mbox_isr_yield_val);
                if (status) {
                        ath6kl_err("bmi_write_memory for yield limit failed\n");
                        goto out;
                }
        }

out:
        return status;
}

static int ath6kl_target_config_wlan_params(struct ath6kl *ar, int idx)
{
        int ret;

        /*
         * Configure the device for rx dot11 header rules. "0,0" are the
         * default values. Required if checksum offload is needed. Set
         * RxMetaVersion to 2.
         */
        ret = ath6kl_wmi_set_rx_frame_format_cmd(ar->wmi, idx,
                                                 ar->rx_meta_ver, 0, 0);
        if (ret) {
                ath6kl_err("unable to set the rx frame format: %d\n", ret);
                return ret;
        }

        if (ar->conf_flags & ATH6KL_CONF_IGNORE_PS_FAIL_EVT_IN_SCAN) {
                ret = ath6kl_wmi_pmparams_cmd(ar->wmi, idx, 0, 1, 0, 0, 1,
                                              IGNORE_PS_FAIL_DURING_SCAN);
                if (ret) {
                        ath6kl_err("unable to set power save fail event policy: %d\n",
                                   ret);
                        return ret;
                }
        }

        if (!(ar->conf_flags & ATH6KL_CONF_IGNORE_ERP_BARKER)) {
                ret = ath6kl_wmi_set_lpreamble_cmd(ar->wmi, idx, 0,
                                                   WMI_FOLLOW_BARKER_IN_ERP);
                if (ret) {
                        ath6kl_err("unable to set barker preamble policy: %d\n",
                                   ret);
                        return ret;
                }
        }

        ret = ath6kl_wmi_set_keepalive_cmd(ar->wmi, idx,
                                           WLAN_CONFIG_KEEP_ALIVE_INTERVAL);
        if (ret) {
                ath6kl_err("unable to set keep alive interval: %d\n", ret);
                return ret;
        }

        ret = ath6kl_wmi_disctimeout_cmd(ar->wmi, idx,
                                         WLAN_CONFIG_DISCONNECT_TIMEOUT);
        if (ret) {
                ath6kl_err("unable to set disconnect timeout: %d\n", ret);
                return ret;
        }

        if (!(ar->conf_flags & ATH6KL_CONF_ENABLE_TX_BURST)) {
                ret = ath6kl_wmi_set_wmm_txop(ar->wmi, idx, WMI_TXOP_DISABLED);
                if (ret) {
                        ath6kl_err("unable to set txop bursting: %d\n", ret);
                        return ret;
                }
        }

        if (ar->p2p && (ar->vif_max == 1 || idx)) {
                ret = ath6kl_wmi_info_req_cmd(ar->wmi, idx,
                                              P2P_FLAG_CAPABILITIES_REQ |
                                              P2P_FLAG_MACADDR_REQ |
                                              P2P_FLAG_HMODEL_REQ);
                if (ret) {
                        ath6kl_dbg(ATH6KL_DBG_TRC,
                                   "failed to request P2P capabilities (%d) - assuming P2P not supported\n",
                                   ret);
                        ar->p2p = false;
                }
        }

        if (ar->p2p && (ar->vif_max == 1 || idx)) {
                /* Enable Probe Request reporting for P2P */
                ret = ath6kl_wmi_probe_report_req_cmd(ar->wmi, idx, true);
                if (ret) {
                        ath6kl_dbg(ATH6KL_DBG_TRC,
                                   "failed to enable Probe Request reporting (%d)\n",
                                   ret);
                }
        }

        return ret;
}

int ath6kl_configure_target(struct ath6kl *ar)
{
        u32 param, ram_reserved_size;
        u8 fw_iftype, fw_mode = 0, fw_submode = 0;
        int i, status;

        param = !!(ar->conf_flags & ATH6KL_CONF_UART_DEBUG);
        if (ath6kl_bmi_write_hi32(ar, hi_serial_enable, param)) {
                ath6kl_err("bmi_write_memory for uart debug failed\n");
                return -EIO;
        }

        /*
         * Note: Even though the firmware interface type is
         * chosen as BSS_STA for all three interfaces, can
         * be configured to IBSS/AP as long as the fw submode
         * remains normal mode (0 - AP, STA and IBSS). But
         * due to an target assert in firmware only one interface is
         * configured for now.
         */
        fw_iftype = HI_OPTION_FW_MODE_BSS_STA;

        for (i = 0; i < ar->vif_max; i++)
                fw_mode |= fw_iftype << (i * HI_OPTION_FW_MODE_BITS);

        /*
         * Submodes when fw does not support dynamic interface
         * switching:
         *                vif[0] - AP/STA/IBSS
         *                vif[1] - "P2P dev"/"P2P GO"/"P2P Client"
         *                vif[2] - "P2P dev"/"P2P GO"/"P2P Client"
         * Otherwise, All the interface are initialized to p2p dev.
         */

        if (test_bit(ATH6KL_FW_CAPABILITY_STA_P2PDEV_DUPLEX,
                     ar->fw_capabilities)) {
                for (i = 0; i < ar->vif_max; i++)
                        fw_submode |= HI_OPTION_FW_SUBMODE_P2PDEV <<
                                (i * HI_OPTION_FW_SUBMODE_BITS);
        } else {
                for (i = 0; i < ar->max_norm_iface; i++)
                        fw_submode |= HI_OPTION_FW_SUBMODE_NONE <<
                                (i * HI_OPTION_FW_SUBMODE_BITS);

                for (i = ar->max_norm_iface; i < ar->vif_max; i++)
                        fw_submode |= HI_OPTION_FW_SUBMODE_P2PDEV <<
                                (i * HI_OPTION_FW_SUBMODE_BITS);

                if (ar->p2p && ar->vif_max == 1)
                        fw_submode = HI_OPTION_FW_SUBMODE_P2PDEV;
        }

        if (ath6kl_bmi_write_hi32(ar, hi_app_host_interest,
                                  HTC_PROTOCOL_VERSION) != 0) {
                ath6kl_err("bmi_write_memory for htc version failed\n");
                return -EIO;
        }

        /* set the firmware mode to STA/IBSS/AP */
        param = 0;

        if (ath6kl_bmi_read_hi32(ar, hi_option_flag, &param) != 0) {
                ath6kl_err("bmi_read_memory for setting fwmode failed\n");
                return -EIO;
        }

        param |= (ar->vif_max << HI_OPTION_NUM_DEV_SHIFT);
        param |= fw_mode << HI_OPTION_FW_MODE_SHIFT;
        param |= fw_submode << HI_OPTION_FW_SUBMODE_SHIFT;

        param |= (0 << HI_OPTION_MAC_ADDR_METHOD_SHIFT);
        param |= (0 << HI_OPTION_FW_BRIDGE_SHIFT);

        if (ath6kl_bmi_write_hi32(ar, hi_option_flag, param) != 0) {
                ath6kl_err("bmi_write_memory for setting fwmode failed\n");
                return -EIO;
        }

        ath6kl_dbg(ATH6KL_DBG_TRC, "firmware mode set\n");

        /*
         * Hardcode the address use for the extended board data
         * Ideally this should be pre-allocate by the OS at boot time
         * But since it is a new feature and board data is loaded
         * at init time, we have to workaround this from host.
         * It is difficult to patch the firmware boot code,
         * but possible in theory.
         */

        if ((ar->target_type == TARGET_TYPE_AR6003) ||
            (ar->version.target_ver == AR6004_HW_1_3_VERSION) ||
            (ar->version.target_ver == AR6004_HW_3_0_VERSION)) {
                param = ar->hw.board_ext_data_addr;
                ram_reserved_size = ar->hw.reserved_ram_size;

                if (ath6kl_bmi_write_hi32(ar, hi_board_ext_data, param) != 0) {
                        ath6kl_err("bmi_write_memory for hi_board_ext_data failed\n");
                        return -EIO;
                }

                if (ath6kl_bmi_write_hi32(ar, hi_end_ram_reserve_sz,
                                          ram_reserved_size) != 0) {
                        ath6kl_err("bmi_write_memory for hi_end_ram_reserve_sz failed\n");
                        return -EIO;
                }
        }

        /* set the block size for the target */
        if (ath6kl_set_htc_params(ar, MBOX_YIELD_LIMIT, 0))
                /* use default number of control buffers */
                return -EIO;

        /* Configure GPIO AR600x UART */
        status = ath6kl_bmi_write_hi32(ar, hi_dbg_uart_txpin,
                                       ar->hw.uarttx_pin);
        if (status)
                return status;

        /* Only set the baud rate if we're actually doing debug */
        if (ar->conf_flags & ATH6KL_CONF_UART_DEBUG) {
                status = ath6kl_bmi_write_hi32(ar, hi_desired_baud_rate,
                                               ar->hw.uarttx_rate);
                if (status)
                        return status;
        }

        /* Configure target refclk_hz */
        if (ar->hw.refclk_hz != 0) {
                status = ath6kl_bmi_write_hi32(ar, hi_refclk_hz,
                                               ar->hw.refclk_hz);
                if (status)
                        return status;
        }

        return 0;
}

/* firmware upload */
static int ath6kl_get_fw(struct ath6kl *ar, const char *filename,
                         u8 **fw, size_t *fw_len)
{
        const struct firmware *fw_entry;
        int ret;

        ret = request_firmware(&fw_entry, filename, ar->dev);
        if (ret)
                return ret;

        *fw_len = fw_entry->size;
        *fw = kmemdup(fw_entry->data, fw_entry->size, GFP_KERNEL);

        if (*fw == NULL)
                ret = -ENOMEM;

        release_firmware(fw_entry);

        return ret;
}

#ifdef CONFIG_OF
/*
 * Check the device tree for a board-id and use it to construct
 * the pathname to the firmware file.  Used (for now) to find a
 * fallback to the "bdata.bin" file--typically a symlink to the
 * appropriate board-specific file.
 */
static bool check_device_tree(struct ath6kl *ar)
{
        static const char *board_id_prop = "atheros,board-id";
        struct device_node *node;
        char board_filename[64];
        const char *board_id;
        int ret;

        for_each_compatible_node(node, NULL, "atheros,ath6kl") {
                board_id = of_get_property(node, board_id_prop, NULL);
                if (board_id == NULL) {
                        ath6kl_warn("No \"%s\" property on %pOFn node.\n",
                                    board_id_prop, node);
                        continue;
                }
                snprintf(board_filename, sizeof(board_filename),
                         "%s/bdata.%s.bin", ar->hw.fw.dir, board_id);

                ret = ath6kl_get_fw(ar, board_filename, &ar->fw_board,
                                    &ar->fw_board_len);
                if (ret) {
                        ath6kl_err("Failed to get DT board file %s: %d\n",
                                   board_filename, ret);
                        continue;
                }
                of_node_put(node);
                return true;
        }
        return false;
}
#else
static bool check_device_tree(struct ath6kl *ar)
{
        return false;
}
#endif /* CONFIG_OF */

static int ath6kl_fetch_board_file(struct ath6kl *ar)
{
        const char *filename;
        int ret;

        if (ar->fw_board != NULL)
                return 0;

        if (WARN_ON(ar->hw.fw_board == NULL))
                return -EINVAL;

        filename = ar->hw.fw_board;

        ret = ath6kl_get_fw(ar, filename, &ar->fw_board,
                            &ar->fw_board_len);
        if (ret == 0) {
                /* managed to get proper board file */
                return 0;
        }

        if (check_device_tree(ar)) {
                /* got board file from device tree */
                return 0;
        }

        /* there was no proper board file, try to use default instead */
        ath6kl_warn("Failed to get board file %s (%d), trying to find default board file.\n",
                    filename, ret);

        filename = ar->hw.fw_default_board;

        ret = ath6kl_get_fw(ar, filename, &ar->fw_board,
                            &ar->fw_board_len);
        if (ret) {
                ath6kl_err("Failed to get default board file %s: %d\n",
                           filename, ret);
                return ret;
        }

        ath6kl_warn("WARNING! No proper board file was not found, instead using a default board file.\n");
        ath6kl_warn("Most likely your hardware won't work as specified. Install correct board file!\n");

        return 0;
}

static int ath6kl_fetch_otp_file(struct ath6kl *ar)
{
        char filename[100];
        int ret;

        if (ar->fw_otp != NULL)
                return 0;

        if (ar->hw.fw.otp == NULL) {
                ath6kl_dbg(ATH6KL_DBG_BOOT,
                           "no OTP file configured for this hw\n");
                return 0;
        }

        snprintf(filename, sizeof(filename), "%s/%s",
                 ar->hw.fw.dir, ar->hw.fw.otp);

        ret = ath6kl_get_fw(ar, filename, &ar->fw_otp,
                            &ar->fw_otp_len);
        if (ret) {
                ath6kl_err("Failed to get OTP file %s: %d\n",
                           filename, ret);
                return ret;
        }

        return 0;
}

static int ath6kl_fetch_testmode_file(struct ath6kl *ar)
{
        char filename[100];
        int ret;

        if (ar->testmode == 0)
                return 0;

        ath6kl_dbg(ATH6KL_DBG_BOOT, "testmode %d\n", ar->testmode);

        if (ar->testmode == 2) {
                if (ar->hw.fw.utf == NULL) {
                        ath6kl_warn("testmode 2 not supported\n");
                        return -EOPNOTSUPP;
                }

                snprintf(filename, sizeof(filename), "%s/%s",
                         ar->hw.fw.dir, ar->hw.fw.utf);
        } else {
                if (ar->hw.fw.tcmd == NULL) {
                        ath6kl_warn("testmode 1 not supported\n");
                        return -EOPNOTSUPP;
                }

                snprintf(filename, sizeof(filename), "%s/%s",
                         ar->hw.fw.dir, ar->hw.fw.tcmd);
        }

        set_bit(TESTMODE, &ar->flag);

        ret = ath6kl_get_fw(ar, filename, &ar->fw, &ar->fw_len);
        if (ret) {
                ath6kl_err("Failed to get testmode %d firmware file %s: %d\n",
                           ar->testmode, filename, ret);
                return ret;
        }

        return 0;
}

static int ath6kl_fetch_fw_file(struct ath6kl *ar)
{
        char filename[100];
        int ret;

        if (ar->fw != NULL)
                return 0;

        /* FIXME: remove WARN_ON() as we won't support FW API 1 for long */
        if (WARN_ON(ar->hw.fw.fw == NULL))
                return -EINVAL;

        snprintf(filename, sizeof(filename), "%s/%s",
                 ar->hw.fw.dir, ar->hw.fw.fw);

        ret = ath6kl_get_fw(ar, filename, &ar->fw, &ar->fw_len);
        if (ret) {
                ath6kl_err("Failed to get firmware file %s: %d\n",
                           filename, ret);
                return ret;
        }

        return 0;
}

static int ath6kl_fetch_patch_file(struct ath6kl *ar)
{
        char filename[100];
        int ret;

        if (ar->fw_patch != NULL)
                return 0;

        if (ar->hw.fw.patch == NULL)
                return 0;

        snprintf(filename, sizeof(filename), "%s/%s",
                 ar->hw.fw.dir, ar->hw.fw.patch);

        ret = ath6kl_get_fw(ar, filename, &ar->fw_patch,
                            &ar->fw_patch_len);
        if (ret) {
                ath6kl_err("Failed to get patch file %s: %d\n",
                           filename, ret);
                return ret;
        }

        return 0;
}

static int ath6kl_fetch_testscript_file(struct ath6kl *ar)
{
        char filename[100];
        int ret;

        if (ar->testmode != 2)
                return 0;

        if (ar->fw_testscript != NULL)
                return 0;

        if (ar->hw.fw.testscript == NULL)
                return 0;

        snprintf(filename, sizeof(filename), "%s/%s",
                 ar->hw.fw.dir, ar->hw.fw.testscript);

        ret = ath6kl_get_fw(ar, filename, &ar->fw_testscript,
                                &ar->fw_testscript_len);
        if (ret) {
                ath6kl_err("Failed to get testscript file %s: %d\n",
                           filename, ret);
                return ret;
        }

        return 0;
}

static int ath6kl_fetch_fw_api1(struct ath6kl *ar)
{
        int ret;

        ret = ath6kl_fetch_otp_file(ar);
        if (ret)
                return ret;

        ret = ath6kl_fetch_fw_file(ar);
        if (ret)
                return ret;

        ret = ath6kl_fetch_patch_file(ar);
        if (ret)
                return ret;

        ret = ath6kl_fetch_testscript_file(ar);
        if (ret)
                return ret;

        return 0;
}

static int ath6kl_fetch_fw_apin(struct ath6kl *ar, const char *name)
{
        size_t magic_len, len, ie_len;
        const struct firmware *fw;
        struct ath6kl_fw_ie *hdr;
        char filename[100];
        const u8 *data;
        int ret, ie_id, i, index, bit;
        __le32 *val;

        snprintf(filename, sizeof(filename), "%s/%s", ar->hw.fw.dir, name);

        ret = request_firmware(&fw, filename, ar->dev);
        if (ret) {
                ath6kl_err("Failed request firmware, rv: %d\n", ret);
                return ret;
        }

        data = fw->data;
        len = fw->size;

        /* magic also includes the null byte, check that as well */
        magic_len = strlen(ATH6KL_FIRMWARE_MAGIC) + 1;

        if (len < magic_len) {
                ath6kl_err("Magic length is invalid, len: %zd  magic_len: %zd\n",
                           len, magic_len);
                ret = -EINVAL;
                goto out;
        }

        if (memcmp(data, ATH6KL_FIRMWARE_MAGIC, magic_len) != 0) {
                ath6kl_err("Magic is invalid, magic_len: %zd\n",
                           magic_len);
                ret = -EINVAL;
                goto out;
        }

        len -= magic_len;
        data += magic_len;

        /* loop elements */
        while (len > sizeof(struct ath6kl_fw_ie)) {
                /* hdr is unaligned! */
                hdr = (struct ath6kl_fw_ie *) data;

                ie_id = le32_to_cpup(&hdr->id);
                ie_len = le32_to_cpup(&hdr->len);

                len -= sizeof(*hdr);
                data += sizeof(*hdr);

                ath6kl_dbg(ATH6KL_DBG_BOOT, "ie-id: %d  len: %zd (0x%zx)\n",
                           ie_id, ie_len, ie_len);

                if (len < ie_len) {
                        ath6kl_err("IE len is invalid, len: %zd  ie_len: %zd  ie-id: %d\n",
                                   len, ie_len, ie_id);
                        ret = -EINVAL;
                        goto out;
                }

                switch (ie_id) {
                case ATH6KL_FW_IE_FW_VERSION:
                        strscpy(ar->wiphy->fw_version, data,
                                min(sizeof(ar->wiphy->fw_version), ie_len+1));

                        ath6kl_dbg(ATH6KL_DBG_BOOT,
                                   "found fw version %s\n",
                                    ar->wiphy->fw_version);
                        break;
                case ATH6KL_FW_IE_OTP_IMAGE:
                        ath6kl_dbg(ATH6KL_DBG_BOOT, "found otp image ie (%zd B)\n",
                                   ie_len);

                        ar->fw_otp = kmemdup(data, ie_len, GFP_KERNEL);

                        if (ar->fw_otp == NULL) {
                                ath6kl_err("fw_otp cannot be allocated\n");
                                ret = -ENOMEM;
                                goto out;
                        }

                        ar->fw_otp_len = ie_len;
                        break;
                case ATH6KL_FW_IE_FW_IMAGE:
                        ath6kl_dbg(ATH6KL_DBG_BOOT, "found fw image ie (%zd B)\n",
                                   ie_len);

                        /* in testmode we already might have a fw file */
                        if (ar->fw != NULL)
                                break;

                        ar->fw = vmalloc(ie_len);

                        if (ar->fw == NULL) {
                                ath6kl_err("fw storage cannot be allocated, len: %zd\n", ie_len);
                                ret = -ENOMEM;
                                goto out;
                        }

                        memcpy(ar->fw, data, ie_len);
                        ar->fw_len = ie_len;
                        break;
                case ATH6KL_FW_IE_PATCH_IMAGE:
                        ath6kl_dbg(ATH6KL_DBG_BOOT, "found patch image ie (%zd B)\n",
                                   ie_len);

                        ar->fw_patch = kmemdup(data, ie_len, GFP_KERNEL);

                        if (ar->fw_patch == NULL) {
                                ath6kl_err("fw_patch storage cannot be allocated, len: %zd\n", ie_len);
                                ret = -ENOMEM;
                                goto out;
                        }

                        ar->fw_patch_len = ie_len;
                        break;
                case ATH6KL_FW_IE_RESERVED_RAM_SIZE:
                        val = (__le32 *) data;
                        ar->hw.reserved_ram_size = le32_to_cpup(val);

                        ath6kl_dbg(ATH6KL_DBG_BOOT,
                                   "found reserved ram size ie %d\n",
                                   ar->hw.reserved_ram_size);
                        break;
                case ATH6KL_FW_IE_CAPABILITIES:
                        ath6kl_dbg(ATH6KL_DBG_BOOT,
                                   "found firmware capabilities ie (%zd B)\n",
                                   ie_len);

                        for (i = 0; i < ATH6KL_FW_CAPABILITY_MAX; i++) {
                                index = i / 8;
                                bit = i % 8;

                                if (index == ie_len)
                                        break;

                                if (data[index] & (1 << bit))
                                        __set_bit(i, ar->fw_capabilities);
                        }

                        ath6kl_dbg_dump(ATH6KL_DBG_BOOT, "capabilities", "",
                                        ar->fw_capabilities,
                                        sizeof(ar->fw_capabilities));
                        break;
                case ATH6KL_FW_IE_PATCH_ADDR:
                        if (ie_len != sizeof(*val))
                                break;

                        val = (__le32 *) data;
                        ar->hw.dataset_patch_addr = le32_to_cpup(val);

                        ath6kl_dbg(ATH6KL_DBG_BOOT,
                                   "found patch address ie 0x%x\n",
                                   ar->hw.dataset_patch_addr);
                        break;
                case ATH6KL_FW_IE_BOARD_ADDR:
                        if (ie_len != sizeof(*val))
                                break;

                        val = (__le32 *) data;
                        ar->hw.board_addr = le32_to_cpup(val);

                        ath6kl_dbg(ATH6KL_DBG_BOOT,
                                   "found board address ie 0x%x\n",
                                   ar->hw.board_addr);
                        break;
                case ATH6KL_FW_IE_VIF_MAX:
                        if (ie_len != sizeof(*val))
                                break;

                        val = (__le32 *) data;
                        ar->vif_max = min_t(unsigned int, le32_to_cpup(val),
                                            ATH6KL_VIF_MAX);

                        if (ar->vif_max > 1 && !ar->p2p)
                                ar->max_norm_iface = 2;

                        ath6kl_dbg(ATH6KL_DBG_BOOT,
                                   "found vif max ie %d\n", ar->vif_max);
                        break;
                default:
                        ath6kl_dbg(ATH6KL_DBG_BOOT, "Unknown fw ie: %u\n",
                                   le32_to_cpup(&hdr->id));
                        break;
                }

                len -= ie_len;
                data += ie_len;
        }

        ret = 0;
out:
        release_firmware(fw);

        return ret;
}

int ath6kl_init_fetch_firmwares(struct ath6kl *ar)
{
        int ret;

        ret = ath6kl_fetch_board_file(ar);
        if (ret)
                return ret;

        ret = ath6kl_fetch_testmode_file(ar);
        if (ret)
                return ret;

        ret = ath6kl_fetch_fw_apin(ar, ATH6KL_FW_API5_FILE);
        if (ret == 0) {
                ar->fw_api = 5;
                goto out;
        }

        ret = ath6kl_fetch_fw_apin(ar, ATH6KL_FW_API4_FILE);
        if (ret == 0) {
                ar->fw_api = 4;
                goto out;
        }

        ret = ath6kl_fetch_fw_apin(ar, ATH6KL_FW_API3_FILE);
        if (ret == 0) {
                ar->fw_api = 3;
                goto out;
        }

        ret = ath6kl_fetch_fw_apin(ar, ATH6KL_FW_API2_FILE);
        if (ret == 0) {
                ar->fw_api = 2;
                goto out;
        }

        ret = ath6kl_fetch_fw_api1(ar);
        if (ret)
                return ret;

        ar->fw_api = 1;

out:
        ath6kl_dbg(ATH6KL_DBG_BOOT, "using fw api %d\n", ar->fw_api);

        return 0;
}

static int ath6kl_upload_board_file(struct ath6kl *ar)
{
        u32 board_address, board_ext_address, param;
        u32 board_data_size, board_ext_data_size;
        int ret;

        if (WARN_ON(ar->fw_board == NULL))
                return -ENOENT;

        /*
         * Determine where in Target RAM to write Board Data.
         * For AR6004, host determine Target RAM address for
         * writing board data.
         */
        if (ar->hw.board_addr != 0) {
                board_address = ar->hw.board_addr;
                ath6kl_bmi_write_hi32(ar, hi_board_data,
                                      board_address);
        } else {
                ret = ath6kl_bmi_read_hi32(ar, hi_board_data, &board_address);
                if (ret) {
                        ath6kl_err("Failed to get board file target address.\n");
                        return ret;
                }
        }

        /* determine where in target ram to write extended board data */
        ret = ath6kl_bmi_read_hi32(ar, hi_board_ext_data, &board_ext_address);
        if (ret) {
                ath6kl_err("Failed to get extended board file target address.\n");
                return ret;
        }

        if (ar->target_type == TARGET_TYPE_AR6003 &&
            board_ext_address == 0) {
                ath6kl_err("Failed to get board file target address.\n");
                return -EINVAL;
        }

        switch (ar->target_type) {
        case TARGET_TYPE_AR6003:
                board_data_size = AR6003_BOARD_DATA_SZ;
                board_ext_data_size = AR6003_BOARD_EXT_DATA_SZ;
                if (ar->fw_board_len > (board_data_size + board_ext_data_size))
                        board_ext_data_size = AR6003_BOARD_EXT_DATA_SZ_V2;
                break;
        case TARGET_TYPE_AR6004:
                board_data_size = AR6004_BOARD_DATA_SZ;
                board_ext_data_size = AR6004_BOARD_EXT_DATA_SZ;
                break;
        default:
                WARN_ON(1);
                return -EINVAL;
        }

        if (board_ext_address &&
            ar->fw_board_len == (board_data_size + board_ext_data_size)) {
                /* write extended board data */
                ath6kl_dbg(ATH6KL_DBG_BOOT,
                           "writing extended board data to 0x%x (%d B)\n",
                           board_ext_address, board_ext_data_size);

                ret = ath6kl_bmi_write(ar, board_ext_address,
                                       ar->fw_board + board_data_size,
                                       board_ext_data_size);
                if (ret) {
                        ath6kl_err("Failed to write extended board data: %d\n",
                                   ret);
                        return ret;
                }

                /* record that extended board data is initialized */
                param = (board_ext_data_size << 16) | 1;

                ath6kl_bmi_write_hi32(ar, hi_board_ext_data_config, param);
        }

        if (ar->fw_board_len < board_data_size) {
                ath6kl_err("Too small board file: %zu\n", ar->fw_board_len);
                ret = -EINVAL;
                return ret;
        }

        ath6kl_dbg(ATH6KL_DBG_BOOT, "writing board file to 0x%x (%d B)\n",
                   board_address, board_data_size);

        ret = ath6kl_bmi_write(ar, board_address, ar->fw_board,
                               board_data_size);

        if (ret) {
                ath6kl_err("Board file bmi write failed: %d\n", ret);
                return ret;
        }

        /* record the fact that Board Data IS initialized */
        if ((ar->version.target_ver == AR6004_HW_1_3_VERSION) ||
            (ar->version.target_ver == AR6004_HW_3_0_VERSION))
                param = board_data_size;
        else
                param = 1;

        ath6kl_bmi_write_hi32(ar, hi_board_data_initialized, param);

        return ret;
}

static int ath6kl_upload_otp(struct ath6kl *ar)
{
        u32 address, param;
        bool from_hw = false;
        int ret;

        if (ar->fw_otp == NULL)
                return 0;

        address = ar->hw.app_load_addr;

        ath6kl_dbg(ATH6KL_DBG_BOOT, "writing otp to 0x%x (%zd B)\n", address,
                   ar->fw_otp_len);

        ret = ath6kl_bmi_fast_download(ar, address, ar->fw_otp,
                                       ar->fw_otp_len);
        if (ret) {
                ath6kl_err("Failed to upload OTP file: %d\n", ret);
                return ret;
        }

        /* read firmware start address */
        ret = ath6kl_bmi_read_hi32(ar, hi_app_start, &address);

        if (ret) {
                ath6kl_err("Failed to read hi_app_start: %d\n", ret);
                return ret;
        }

        if (ar->hw.app_start_override_addr == 0) {
                ar->hw.app_start_override_addr = address;
                from_hw = true;
        }

        ath6kl_dbg(ATH6KL_DBG_BOOT, "app_start_override_addr%s 0x%x\n",
                   from_hw ? " (from hw)" : "",
                   ar->hw.app_start_override_addr);

        /* execute the OTP code */
        ath6kl_dbg(ATH6KL_DBG_BOOT, "executing OTP at 0x%x\n",
                   ar->hw.app_start_override_addr);
        param = 0;
        ath6kl_bmi_execute(ar, ar->hw.app_start_override_addr, &param);

        return ret;
}

static int ath6kl_upload_firmware(struct ath6kl *ar)
{
        u32 address;
        int ret;

        if (WARN_ON(ar->fw == NULL))
                return 0;

        address = ar->hw.app_load_addr;

        ath6kl_dbg(ATH6KL_DBG_BOOT, "writing firmware to 0x%x (%zd B)\n",
                   address, ar->fw_len);

        ret = ath6kl_bmi_fast_download(ar, address, ar->fw, ar->fw_len);

        if (ret) {
                ath6kl_err("Failed to write firmware: %d\n", ret);
                return ret;
        }

        /*
         * Set starting address for firmware
         * Don't need to setup app_start override addr on AR6004
         */
        if (ar->target_type != TARGET_TYPE_AR6004) {
                address = ar->hw.app_start_override_addr;
                ath6kl_bmi_set_app_start(ar, address);
        }
        return ret;
}

static int ath6kl_upload_patch(struct ath6kl *ar)
{
        u32 address;
        int ret;

        if (ar->fw_patch == NULL)
                return 0;

        address = ar->hw.dataset_patch_addr;

        ath6kl_dbg(ATH6KL_DBG_BOOT, "writing patch to 0x%x (%zd B)\n",
                   address, ar->fw_patch_len);

        ret = ath6kl_bmi_write(ar, address, ar->fw_patch, ar->fw_patch_len);
        if (ret) {
                ath6kl_err("Failed to write patch file: %d\n", ret);
                return ret;
        }

        ath6kl_bmi_write_hi32(ar, hi_dset_list_head, address);

        return 0;
}

static int ath6kl_upload_testscript(struct ath6kl *ar)
{
        u32 address;
        int ret;

        if (ar->testmode != 2)
                return 0;

        if (ar->fw_testscript == NULL)
                return 0;

        address = ar->hw.testscript_addr;

        ath6kl_dbg(ATH6KL_DBG_BOOT, "writing testscript to 0x%x (%zd B)\n",
                   address, ar->fw_testscript_len);

        ret = ath6kl_bmi_write(ar, address, ar->fw_testscript,
                ar->fw_testscript_len);
        if (ret) {
                ath6kl_err("Failed to write testscript file: %d\n", ret);
                return ret;
        }

        ath6kl_bmi_write_hi32(ar, hi_ota_testscript, address);

        if ((ar->version.target_ver != AR6004_HW_1_3_VERSION) &&
            (ar->version.target_ver != AR6004_HW_3_0_VERSION))
                ath6kl_bmi_write_hi32(ar, hi_end_ram_reserve_sz, 4096);

        ath6kl_bmi_write_hi32(ar, hi_test_apps_related, 1);

        return 0;
}

static int ath6kl_init_upload(struct ath6kl *ar)
{
        u32 param, options, sleep, address;
        int status = 0;

        if (ar->target_type != TARGET_TYPE_AR6003 &&
            ar->target_type != TARGET_TYPE_AR6004)
                return -EINVAL;

        /* temporarily disable system sleep */
        address = MBOX_BASE_ADDRESS + LOCAL_SCRATCH_ADDRESS;
        status = ath6kl_bmi_reg_read(ar, address, &param);
        if (status)
                return status;

        options = param;

        param |= ATH6KL_OPTION_SLEEP_DISABLE;
        status = ath6kl_bmi_reg_write(ar, address, param);
        if (status)
                return status;

        address = RTC_BASE_ADDRESS + SYSTEM_SLEEP_ADDRESS;
        status = ath6kl_bmi_reg_read(ar, address, &param);
        if (status)
                return status;

        sleep = param;

        param |= SM(SYSTEM_SLEEP_DISABLE, 1);
        status = ath6kl_bmi_reg_write(ar, address, param);
        if (status)
                return status;

        ath6kl_dbg(ATH6KL_DBG_TRC, "old options: %d, old sleep: %d\n",
                   options, sleep);

        /* program analog PLL register */
        /* no need to control 40/44MHz clock on AR6004 */
        if (ar->target_type != TARGET_TYPE_AR6004) {
                status = ath6kl_bmi_reg_write(ar, ATH6KL_ANALOG_PLL_REGISTER,
                                              0xF9104001);

                if (status)
                        return status;

                /* Run at 80/88MHz by default */
                param = SM(CPU_CLOCK_STANDARD, 1);

                address = RTC_BASE_ADDRESS + CPU_CLOCK_ADDRESS;
                status = ath6kl_bmi_reg_write(ar, address, param);
                if (status)
                        return status;
        }

        param = 0;
        address = RTC_BASE_ADDRESS + LPO_CAL_ADDRESS;
        param = SM(LPO_CAL_ENABLE, 1);
        status = ath6kl_bmi_reg_write(ar, address, param);
        if (status)
                return status;

        /* WAR to avoid SDIO CRC err */
        if (ar->hw.flags & ATH6KL_HW_SDIO_CRC_ERROR_WAR) {
                ath6kl_err("temporary war to avoid sdio crc error\n");

                param = 0x28;
                address = GPIO_BASE_ADDRESS + GPIO_PIN9_ADDRESS;
                status = ath6kl_bmi_reg_write(ar, address, param);
                if (status)
                        return status;

                param = 0x20;

                address = GPIO_BASE_ADDRESS + GPIO_PIN10_ADDRESS;
                status = ath6kl_bmi_reg_write(ar, address, param);
                if (status)
                        return status;

                address = GPIO_BASE_ADDRESS + GPIO_PIN11_ADDRESS;
                status = ath6kl_bmi_reg_write(ar, address, param);
                if (status)
                        return status;

                address = GPIO_BASE_ADDRESS + GPIO_PIN12_ADDRESS;
                status = ath6kl_bmi_reg_write(ar, address, param);
                if (status)
                        return status;

                address = GPIO_BASE_ADDRESS + GPIO_PIN13_ADDRESS;
                status = ath6kl_bmi_reg_write(ar, address, param);
                if (status)
                        return status;
        }

        /* write EEPROM data to Target RAM */
        status = ath6kl_upload_board_file(ar);
        if (status)
                return status;

        /* transfer One time Programmable data */
        status = ath6kl_upload_otp(ar);
        if (status)
                return status;

        /* Download Target firmware */
        status = ath6kl_upload_firmware(ar);
        if (status)
                return status;

        status = ath6kl_upload_patch(ar);
        if (status)
                return status;

        /* Download the test script */
        status = ath6kl_upload_testscript(ar);
        if (status)
                return status;

        /* Restore system sleep */
        address = RTC_BASE_ADDRESS + SYSTEM_SLEEP_ADDRESS;
        status = ath6kl_bmi_reg_write(ar, address, sleep);
        if (status)
                return status;

        address = MBOX_BASE_ADDRESS + LOCAL_SCRATCH_ADDRESS;
        param = options | 0x20;
        status = ath6kl_bmi_reg_write(ar, address, param);
        if (status)
                return status;

        return status;
}

int ath6kl_init_hw_params(struct ath6kl *ar)
{
        const struct ath6kl_hw *hw;
        int i;

        for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
                hw = &hw_list[i];

                if (hw->id == ar->version.target_ver)
                        break;
        }

        if (i == ARRAY_SIZE(hw_list)) {
                ath6kl_err("Unsupported hardware version: 0x%x\n",
                           ar->version.target_ver);
                return -EINVAL;
        }

        ar->hw = *hw;

        ath6kl_dbg(ATH6KL_DBG_BOOT,
                   "target_ver 0x%x target_type 0x%x dataset_patch 0x%x app_load_addr 0x%x\n",
                   ar->version.target_ver, ar->target_type,
                   ar->hw.dataset_patch_addr, ar->hw.app_load_addr);
        ath6kl_dbg(ATH6KL_DBG_BOOT,
                   "app_start_override_addr 0x%x board_ext_data_addr 0x%x reserved_ram_size 0x%x",
                   ar->hw.app_start_override_addr, ar->hw.board_ext_data_addr,
                   ar->hw.reserved_ram_size);
        ath6kl_dbg(ATH6KL_DBG_BOOT,
                   "refclk_hz %d uarttx_pin %d",
                   ar->hw.refclk_hz, ar->hw.uarttx_pin);

        return 0;
}

static const char *ath6kl_init_get_hif_name(enum ath6kl_hif_type type)
{
        switch (type) {
        case ATH6KL_HIF_TYPE_SDIO:
                return "sdio";
        case ATH6KL_HIF_TYPE_USB:
                return "usb";
        }

        return NULL;
}


static const struct fw_capa_str_map {
        int id;
        const char *name;
} fw_capa_map[] = {
        { ATH6KL_FW_CAPABILITY_HOST_P2P, "host-p2p" },
        { ATH6KL_FW_CAPABILITY_SCHED_SCAN, "sched-scan" },
        { ATH6KL_FW_CAPABILITY_STA_P2PDEV_DUPLEX, "sta-p2pdev-duplex" },
        { ATH6KL_FW_CAPABILITY_INACTIVITY_TIMEOUT, "inactivity-timeout" },
        { ATH6KL_FW_CAPABILITY_RSN_CAP_OVERRIDE, "rsn-cap-override" },
        { ATH6KL_FW_CAPABILITY_WOW_MULTICAST_FILTER, "wow-mc-filter" },
        { ATH6KL_FW_CAPABILITY_BMISS_ENHANCE, "bmiss-enhance" },
        { ATH6KL_FW_CAPABILITY_SCHED_SCAN_MATCH_LIST, "sscan-match-list" },
        { ATH6KL_FW_CAPABILITY_RSSI_SCAN_THOLD, "rssi-scan-thold" },
        { ATH6KL_FW_CAPABILITY_CUSTOM_MAC_ADDR, "custom-mac-addr" },
        { ATH6KL_FW_CAPABILITY_TX_ERR_NOTIFY, "tx-err-notify" },
        { ATH6KL_FW_CAPABILITY_REGDOMAIN, "regdomain" },
        { ATH6KL_FW_CAPABILITY_SCHED_SCAN_V2, "sched-scan-v2" },
        { ATH6KL_FW_CAPABILITY_HEART_BEAT_POLL, "hb-poll" },
        { ATH6KL_FW_CAPABILITY_64BIT_RATES, "64bit-rates" },
        { ATH6KL_FW_CAPABILITY_AP_INACTIVITY_MINS, "ap-inactivity-mins" },
        { ATH6KL_FW_CAPABILITY_MAP_LP_ENDPOINT, "map-lp-endpoint" },
        { ATH6KL_FW_CAPABILITY_RATETABLE_MCS15, "ratetable-mcs15" },
        { ATH6KL_FW_CAPABILITY_NO_IP_CHECKSUM, "no-ip-checksum" },
};

static const char *ath6kl_init_get_fw_capa_name(unsigned int id)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(fw_capa_map); i++) {
                if (fw_capa_map[i].id == id)
                        return fw_capa_map[i].name;
        }

        return "<unknown>";
}

static void ath6kl_init_get_fwcaps(struct ath6kl *ar, char *buf, size_t buf_len)
{
        u8 *data = (u8 *) ar->fw_capabilities;
        size_t trunc_len, len = 0;
        int i, index, bit;
        char *trunc = "...";

        for (i = 0; i < ATH6KL_FW_CAPABILITY_MAX; i++) {
                index = i / 8;
                bit = i % 8;

                if (index >= sizeof(ar->fw_capabilities) * 4)
                        break;

                if (buf_len - len < 4) {
                        ath6kl_warn("firmware capability buffer too small!\n");

                        /* add "..." to the end of string */
                        trunc_len = strlen(trunc) + 1;
                        memcpy(buf + buf_len - trunc_len, trunc, trunc_len);

                        return;
                }

                if (data[index] & (1 << bit)) {
                        len += scnprintf(buf + len, buf_len - len, "%s,",
                                            ath6kl_init_get_fw_capa_name(i));
                }
        }

        /* overwrite the last comma */
        if (len > 0)
                len--;

        buf[len] = '\0';
}

static int ath6kl_init_hw_reset(struct ath6kl *ar)
{
        ath6kl_dbg(ATH6KL_DBG_BOOT, "cold resetting the device");

        return ath6kl_diag_write32(ar, RESET_CONTROL_ADDRESS,
                                   cpu_to_le32(RESET_CONTROL_COLD_RST));
}

static int __ath6kl_init_hw_start(struct ath6kl *ar)
{
        long timeleft;
        int ret, i;
        char buf[200];

        ath6kl_dbg(ATH6KL_DBG_BOOT, "hw start\n");

        ret = ath6kl_hif_power_on(ar);
        if (ret)
                return ret;

        ret = ath6kl_configure_target(ar);
        if (ret)
                goto err_power_off;

        ret = ath6kl_init_upload(ar);
        if (ret)
                goto err_power_off;

        /* Do we need to finish the BMI phase */
        ret = ath6kl_bmi_done(ar);
        if (ret)
                goto err_power_off;

        /*
         * The reason we have to wait for the target here is that the
         * driver layer has to init BMI in order to set the host block
         * size.
         */
        ret = ath6kl_htc_wait_target(ar->htc_target);

        if (ret == -ETIMEDOUT) {
                /*
                 * Most likely USB target is in odd state after reboot and
                 * needs a reset. A cold reset makes the whole device
                 * disappear from USB bus and initialisation starts from
                 * beginning.
                 */
                ath6kl_warn("htc wait target timed out, resetting device\n");
                ath6kl_init_hw_reset(ar);
                goto err_power_off;
        } else if (ret) {
                ath6kl_err("htc wait target failed: %d\n", ret);
                goto err_power_off;
        }

        ret = ath6kl_init_service_ep(ar);
        if (ret) {
                ath6kl_err("Endpoint service initialization failed: %d\n", ret);
                goto err_cleanup_scatter;
        }

        /* setup credit distribution */
        ath6kl_htc_credit_setup(ar->htc_target, &ar->credit_state_info);

        /* start HTC */
        ret = ath6kl_htc_start(ar->htc_target);
        if (ret) {
                /* FIXME: call this */
                ath6kl_cookie_cleanup(ar);
                goto err_cleanup_scatter;
        }

        /* Wait for Wmi event to be ready */
        timeleft = wait_event_interruptible_timeout(ar->event_wq,
                                                    test_bit(WMI_READY,
                                                             &ar->flag),
                                                    WMI_TIMEOUT);
        if (timeleft <= 0) {
                clear_bit(WMI_READY, &ar->flag);
                ath6kl_err("wmi is not ready or wait was interrupted: %ld\n",
                           timeleft);
                ret = -EIO;
                goto err_htc_stop;
        }

        ath6kl_dbg(ATH6KL_DBG_BOOT, "firmware booted\n");

        if (test_and_clear_bit(FIRST_BOOT, &ar->flag)) {
                ath6kl_info("%s %s fw %s api %d%s\n",
                            ar->hw.name,
                            ath6kl_init_get_hif_name(ar->hif_type),
                            ar->wiphy->fw_version,
                            ar->fw_api,
                            test_bit(TESTMODE, &ar->flag) ? " testmode" : "");
                ath6kl_init_get_fwcaps(ar, buf, sizeof(buf));
                ath6kl_info("firmware supports: %s\n", buf);
        }

        if (ar->version.abi_ver != ATH6KL_ABI_VERSION) {
                ath6kl_err("abi version mismatch: host(0x%x), target(0x%x)\n",
                           ATH6KL_ABI_VERSION, ar->version.abi_ver);
                ret = -EIO;
                goto err_htc_stop;
        }

        ath6kl_dbg(ATH6KL_DBG_TRC, "%s: wmi is ready\n", __func__);

        /* communicate the wmi protocol verision to the target */
        /* FIXME: return error */
        if ((ath6kl_set_host_app_area(ar)) != 0)
                ath6kl_err("unable to set the host app area\n");

        for (i = 0; i < ar->vif_max; i++) {
                ret = ath6kl_target_config_wlan_params(ar, i);
                if (ret)
                        goto err_htc_stop;
        }

        return 0;

err_htc_stop:
        ath6kl_htc_stop(ar->htc_target);
err_cleanup_scatter:
        ath6kl_hif_cleanup_scatter(ar);
err_power_off:
        ath6kl_hif_power_off(ar);

        return ret;
}

int ath6kl_init_hw_start(struct ath6kl *ar)
{
        int err;

        err = __ath6kl_init_hw_start(ar);
        if (err)
                return err;
        ar->state = ATH6KL_STATE_ON;
        return 0;
}

static int __ath6kl_init_hw_stop(struct ath6kl *ar)
{
        int ret;

        ath6kl_dbg(ATH6KL_DBG_BOOT, "hw stop\n");

        ath6kl_htc_stop(ar->htc_target);

        ath6kl_hif_stop(ar);

        ath6kl_bmi_reset(ar);

        ret = ath6kl_hif_power_off(ar);
        if (ret)
                ath6kl_warn("failed to power off hif: %d\n", ret);

        return 0;
}

int ath6kl_init_hw_stop(struct ath6kl *ar)
{
        int err;

        err = __ath6kl_init_hw_stop(ar);
        if (err)
                return err;
        ar->state = ATH6KL_STATE_OFF;
        return 0;
}

void ath6kl_init_hw_restart(struct ath6kl *ar)
{
        clear_bit(WMI_READY, &ar->flag);

        ath6kl_cfg80211_stop_all(ar);

        if (__ath6kl_init_hw_stop(ar)) {
                ath6kl_dbg(ATH6KL_DBG_RECOVERY, "Failed to stop during fw error recovery\n");
                return;
        }

        if (__ath6kl_init_hw_start(ar)) {
                ath6kl_dbg(ATH6KL_DBG_RECOVERY, "Failed to restart during fw error recovery\n");
                return;
        }
}

void ath6kl_stop_txrx(struct ath6kl *ar)
{
        struct ath6kl_vif *vif, *tmp_vif;
        int i;

        set_bit(DESTROY_IN_PROGRESS, &ar->flag);

        if (down_interruptible(&ar->sem)) {
                ath6kl_err("down_interruptible failed\n");
                return;
        }

        for (i = 0; i < AP_MAX_NUM_STA; i++)
                aggr_reset_state(ar->sta_list[i].aggr_conn);

        spin_lock_bh(&ar->list_lock);
        list_for_each_entry_safe(vif, tmp_vif, &ar->vif_list, list) {
                list_del(&vif->list);
                spin_unlock_bh(&ar->list_lock);
                ath6kl_cfg80211_vif_stop(vif, test_bit(WMI_READY, &ar->flag));
                rtnl_lock();
                wiphy_lock(ar->wiphy);
                ath6kl_cfg80211_vif_cleanup(vif);
                wiphy_unlock(ar->wiphy);
                rtnl_unlock();
                spin_lock_bh(&ar->list_lock);
        }
        spin_unlock_bh(&ar->list_lock);

        clear_bit(WMI_READY, &ar->flag);

        if (ar->fw_recovery.enable)
                timer_delete_sync(&ar->fw_recovery.hb_timer);

        /*
         * After wmi_shudown all WMI events will be dropped. We
         * need to cleanup the buffers allocated in AP mode and
         * give disconnect notification to stack, which usually
         * happens in the disconnect_event. Simulate the disconnect
         * event by calling the function directly. Sometimes
         * disconnect_event will be received when the debug logs
         * are collected.
         */
        ath6kl_wmi_shutdown(ar->wmi);

        clear_bit(WMI_ENABLED, &ar->flag);
        if (ar->htc_target) {
                ath6kl_dbg(ATH6KL_DBG_TRC, "%s: shut down htc\n", __func__);
                ath6kl_htc_stop(ar->htc_target);
        }

        /*
         * Try to reset the device if we can. The driver may have been
         * configure NOT to reset the target during a debug session.
         */
        ath6kl_init_hw_reset(ar);

        up(&ar->sem);
}
EXPORT_SYMBOL(ath6kl_stop_txrx);







































   35 







   35 












































  307 





  307 





   70 





  115 




  101 











   55 
   55 


   85 

   31 
















   24 




   24 


   24 






















  109 
   91 










   14 

   16 


   10 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
// SPDX-License-Identifier: GPL-2.0
/*
 *  Floating proportions with flexible aging period
 *
 *   Copyright (C) 2011, SUSE, Jan Kara <jack@suse.cz>
 *
 * The goal of this code is: Given different types of event, measure proportion
 * of each type of event over time. The proportions are measured with
 * exponentially decaying history to give smooth transitions. A formula
 * expressing proportion of event of type 'j' is:
 *
 *   p_{j} = (\Sum_{i>=0} x_{i,j}/2^{i+1})/(\Sum_{i>=0} x_i/2^{i+1})
 *
 * Where x_{i,j} is j's number of events in i-th last time period and x_i is
 * total number of events in i-th last time period.
 *
 * Note that p_{j}'s are normalised, i.e.
 *
 *   \Sum_{j} p_{j} = 1,
 *
 * This formula can be straightforwardly computed by maintaining denominator
 * (let's call it 'd') and for each event type its numerator (let's call it
 * 'n_j'). When an event of type 'j' happens, we simply need to do:
 *   n_j++; d++;
 *
 * When a new period is declared, we could do:
 *   d /= 2
 *   for each j
 *     n_j /= 2
 *
 * To avoid iteration over all event types, we instead shift numerator of event
 * j lazily when someone asks for a proportion of event j or when event j
 * occurs. This can bit trivially implemented by remembering last period in
 * which something happened with proportion of type j.
 */
#include <linux/flex_proportions.h>

int fprop_global_init(struct fprop_global *p, gfp_t gfp)
{
        int err;

        p->period = 0;
        /* Use 1 to avoid dealing with periods with 0 events... */
        err = percpu_counter_init(&p->events, 1, gfp);
        if (err)
                return err;
        seqcount_init(&p->sequence);
        return 0;
}

void fprop_global_destroy(struct fprop_global *p)
{
        percpu_counter_destroy(&p->events);
}

/*
 * Declare @periods new periods. It is upto the caller to make sure period
 * transitions cannot happen in parallel.
 *
 * The function returns true if the proportions are still defined and false
 * if aging zeroed out all events. This can be used to detect whether declaring
 * further periods has any effect.
 */
bool fprop_new_period(struct fprop_global *p, int periods)
{
        s64 events = percpu_counter_sum(&p->events);

        /*
         * Don't do anything if there are no events.
         */
        if (events <= 1)
                return false;
        preempt_disable_nested();
        write_seqcount_begin(&p->sequence);
        if (periods < 64)
                events -= events >> periods;
        /* Use addition to avoid losing events happening between sum and set */
        percpu_counter_add(&p->events, -events);
        p->period += periods;
        write_seqcount_end(&p->sequence);
        preempt_enable_nested();

        return true;
}

/*
 * ---- PERCPU ----
 */
#define PROP_BATCH (8*(1+ilog2(nr_cpu_ids)))

int fprop_local_init_percpu(struct fprop_local_percpu *pl, gfp_t gfp)
{
        int err;

        err = percpu_counter_init(&pl->events, 0, gfp);
        if (err)
                return err;
        pl->period = 0;
        raw_spin_lock_init(&pl->lock);
        return 0;
}

void fprop_local_destroy_percpu(struct fprop_local_percpu *pl)
{
        percpu_counter_destroy(&pl->events);
}

static void fprop_reflect_period_percpu(struct fprop_global *p,
                                        struct fprop_local_percpu *pl)
{
        unsigned int period = p->period;
        unsigned long flags;

        /* Fast path - period didn't change */
        if (pl->period == period)
                return;
        raw_spin_lock_irqsave(&pl->lock, flags);
        /* Someone updated pl->period while we were spinning? */
        if (pl->period >= period) {
                raw_spin_unlock_irqrestore(&pl->lock, flags);
                return;
        }
        /* Aging zeroed our fraction? */
        if (period - pl->period < BITS_PER_LONG) {
                s64 val = percpu_counter_read(&pl->events);

                if (val < (nr_cpu_ids * PROP_BATCH))
                        val = percpu_counter_sum(&pl->events);

                percpu_counter_add_batch(&pl->events,
                        -val + (val >> (period-pl->period)), PROP_BATCH);
        } else
                percpu_counter_set(&pl->events, 0);
        pl->period = period;
        raw_spin_unlock_irqrestore(&pl->lock, flags);
}

/* Event of type pl happened */
void __fprop_add_percpu(struct fprop_global *p, struct fprop_local_percpu *pl,
                long nr)
{
        fprop_reflect_period_percpu(p, pl);
        percpu_counter_add_batch(&pl->events, nr, PROP_BATCH);
        percpu_counter_add(&p->events, nr);
}

void fprop_fraction_percpu(struct fprop_global *p,
                           struct fprop_local_percpu *pl,
                           unsigned long *numerator, unsigned long *denominator)
{
        unsigned int seq;
        s64 num, den;

        do {
                seq = read_seqcount_begin(&p->sequence);
                fprop_reflect_period_percpu(p, pl);
                num = percpu_counter_read_positive(&pl->events);
                den = percpu_counter_read_positive(&p->events);
        } while (read_seqcount_retry(&p->sequence, seq));

        /*
         * Make fraction <= 1 and denominator > 0 even in presence of percpu
         * counter errors
         */
        if (den <= num) {
                if (num)
                        den = num;
                else
                        den = 1;
        }
        *denominator = den;
        *numerator = num;
}

/*
 * Like __fprop_add_percpu() except that event is counted only if the given
 * type has fraction smaller than @max_frac/FPROP_FRAC_BASE
 */
void __fprop_add_percpu_max(struct fprop_global *p,
                struct fprop_local_percpu *pl, int max_frac, long nr)
{
        if (unlikely(max_frac < FPROP_FRAC_BASE)) {
                unsigned long numerator, denominator;
                s64 tmp;

                fprop_fraction_percpu(p, pl, &numerator, &denominator);
                /* Adding 'nr' to fraction exceeds max_frac/FPROP_FRAC_BASE? */
                tmp = (u64)denominator * max_frac -
                                        ((u64)numerator << FPROP_FRAC_SHIFT);
                if (tmp < 0) {
                        /* Maximum fraction already exceeded? */
                        return;
                } else if (tmp < nr * (FPROP_FRAC_BASE - max_frac)) {
                        /* Add just enough for the fraction to saturate */
                        nr = div_u64(tmp + FPROP_FRAC_BASE - max_frac - 1,
                                        FPROP_FRAC_BASE - max_frac);
                }
        }

        __fprop_add_percpu(p, pl, nr);
}





































    1 

























































































































































































































































































































































































































































































































































































































































































































































































    9 





































    1 




    8 














































































































    1 


    8 


























    1 







































































































    8 





































































































































    7 











   11 
   11 






    1 


    7 


    1 









    1 





    3 




    7 
    3 


    7 







    6 
    6 
    4 
    4 




    6 
    6 
    5 
    5 




    4 
    4 
    3 




    7 
    7 
    4 
    4 















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Glue Code for assembler optimized version of Camellia
 *
 * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * Camellia parts based on code by:
 *  Copyright (C) 2006 NTT (Nippon Telegraph and Telephone Corporation)
 */

#include <linux/unaligned.h>
#include <linux/crypto.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>
#include <crypto/algapi.h>

#include "camellia.h"
#include "ecb_cbc_helpers.h"

/* regular block cipher functions */
asmlinkage void __camellia_enc_blk(const void *ctx, u8 *dst, const u8 *src,
                                   bool xor);
EXPORT_SYMBOL_GPL(__camellia_enc_blk);
asmlinkage void camellia_dec_blk(const void *ctx, u8 *dst, const u8 *src);
EXPORT_SYMBOL_GPL(camellia_dec_blk);

/* 2-way parallel cipher functions */
asmlinkage void __camellia_enc_blk_2way(const void *ctx, u8 *dst, const u8 *src,
                                        bool xor);
EXPORT_SYMBOL_GPL(__camellia_enc_blk_2way);
asmlinkage void camellia_dec_blk_2way(const void *ctx, u8 *dst, const u8 *src);
EXPORT_SYMBOL_GPL(camellia_dec_blk_2way);

static void camellia_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        camellia_enc_blk(crypto_tfm_ctx(tfm), dst, src);
}

static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        camellia_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

/* camellia sboxes */
__visible const u64 camellia_sp10011110[256] = {
        0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL,
        0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL,
        0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL,
        0x8500008585858500ULL, 0x5700005757575700ULL, 0x3500003535353500ULL,
        0xea0000eaeaeaea00ULL, 0x0c00000c0c0c0c00ULL, 0xae0000aeaeaeae00ULL,
        0x4100004141414100ULL, 0x2300002323232300ULL, 0xef0000efefefef00ULL,
        0x6b00006b6b6b6b00ULL, 0x9300009393939300ULL, 0x4500004545454500ULL,
        0x1900001919191900ULL, 0xa50000a5a5a5a500ULL, 0x2100002121212100ULL,
        0xed0000edededed00ULL, 0x0e00000e0e0e0e00ULL, 0x4f00004f4f4f4f00ULL,
        0x4e00004e4e4e4e00ULL, 0x1d00001d1d1d1d00ULL, 0x6500006565656500ULL,
        0x9200009292929200ULL, 0xbd0000bdbdbdbd00ULL, 0x8600008686868600ULL,
        0xb80000b8b8b8b800ULL, 0xaf0000afafafaf00ULL, 0x8f00008f8f8f8f00ULL,
        0x7c00007c7c7c7c00ULL, 0xeb0000ebebebeb00ULL, 0x1f00001f1f1f1f00ULL,
        0xce0000cececece00ULL, 0x3e00003e3e3e3e00ULL, 0x3000003030303000ULL,
        0xdc0000dcdcdcdc00ULL, 0x5f00005f5f5f5f00ULL, 0x5e00005e5e5e5e00ULL,
        0xc50000c5c5c5c500ULL, 0x0b00000b0b0b0b00ULL, 0x1a00001a1a1a1a00ULL,
        0xa60000a6a6a6a600ULL, 0xe10000e1e1e1e100ULL, 0x3900003939393900ULL,
        0xca0000cacacaca00ULL, 0xd50000d5d5d5d500ULL, 0x4700004747474700ULL,
        0x5d00005d5d5d5d00ULL, 0x3d00003d3d3d3d00ULL, 0xd90000d9d9d9d900ULL,
        0x0100000101010100ULL, 0x5a00005a5a5a5a00ULL, 0xd60000d6d6d6d600ULL,
        0x5100005151515100ULL, 0x5600005656565600ULL, 0x6c00006c6c6c6c00ULL,
        0x4d00004d4d4d4d00ULL, 0x8b00008b8b8b8b00ULL, 0x0d00000d0d0d0d00ULL,
        0x9a00009a9a9a9a00ULL, 0x6600006666666600ULL, 0xfb0000fbfbfbfb00ULL,
        0xcc0000cccccccc00ULL, 0xb00000b0b0b0b000ULL, 0x2d00002d2d2d2d00ULL,
        0x7400007474747400ULL, 0x1200001212121200ULL, 0x2b00002b2b2b2b00ULL,
        0x2000002020202000ULL, 0xf00000f0f0f0f000ULL, 0xb10000b1b1b1b100ULL,
        0x8400008484848400ULL, 0x9900009999999900ULL, 0xdf0000dfdfdfdf00ULL,
        0x4c00004c4c4c4c00ULL, 0xcb0000cbcbcbcb00ULL, 0xc20000c2c2c2c200ULL,
        0x3400003434343400ULL, 0x7e00007e7e7e7e00ULL, 0x7600007676767600ULL,
        0x0500000505050500ULL, 0x6d00006d6d6d6d00ULL, 0xb70000b7b7b7b700ULL,
        0xa90000a9a9a9a900ULL, 0x3100003131313100ULL, 0xd10000d1d1d1d100ULL,
        0x1700001717171700ULL, 0x0400000404040400ULL, 0xd70000d7d7d7d700ULL,
        0x1400001414141400ULL, 0x5800005858585800ULL, 0x3a00003a3a3a3a00ULL,
        0x6100006161616100ULL, 0xde0000dededede00ULL, 0x1b00001b1b1b1b00ULL,
        0x1100001111111100ULL, 0x1c00001c1c1c1c00ULL, 0x3200003232323200ULL,
        0x0f00000f0f0f0f00ULL, 0x9c00009c9c9c9c00ULL, 0x1600001616161600ULL,
        0x5300005353535300ULL, 0x1800001818181800ULL, 0xf20000f2f2f2f200ULL,
        0x2200002222222200ULL, 0xfe0000fefefefe00ULL, 0x4400004444444400ULL,
        0xcf0000cfcfcfcf00ULL, 0xb20000b2b2b2b200ULL, 0xc30000c3c3c3c300ULL,
        0xb50000b5b5b5b500ULL, 0x7a00007a7a7a7a00ULL, 0x9100009191919100ULL,
        0x2400002424242400ULL, 0x0800000808080800ULL, 0xe80000e8e8e8e800ULL,
        0xa80000a8a8a8a800ULL, 0x6000006060606000ULL, 0xfc0000fcfcfcfc00ULL,
        0x6900006969696900ULL, 0x5000005050505000ULL, 0xaa0000aaaaaaaa00ULL,
        0xd00000d0d0d0d000ULL, 0xa00000a0a0a0a000ULL, 0x7d00007d7d7d7d00ULL,
        0xa10000a1a1a1a100ULL, 0x8900008989898900ULL, 0x6200006262626200ULL,
        0x9700009797979700ULL, 0x5400005454545400ULL, 0x5b00005b5b5b5b00ULL,
        0x1e00001e1e1e1e00ULL, 0x9500009595959500ULL, 0xe00000e0e0e0e000ULL,
        0xff0000ffffffff00ULL, 0x6400006464646400ULL, 0xd20000d2d2d2d200ULL,
        0x1000001010101000ULL, 0xc40000c4c4c4c400ULL, 0x0000000000000000ULL,
        0x4800004848484800ULL, 0xa30000a3a3a3a300ULL, 0xf70000f7f7f7f700ULL,
        0x7500007575757500ULL, 0xdb0000dbdbdbdb00ULL, 0x8a00008a8a8a8a00ULL,
        0x0300000303030300ULL, 0xe60000e6e6e6e600ULL, 0xda0000dadadada00ULL,
        0x0900000909090900ULL, 0x3f00003f3f3f3f00ULL, 0xdd0000dddddddd00ULL,
        0x9400009494949400ULL, 0x8700008787878700ULL, 0x5c00005c5c5c5c00ULL,
        0x8300008383838300ULL, 0x0200000202020200ULL, 0xcd0000cdcdcdcd00ULL,
        0x4a00004a4a4a4a00ULL, 0x9000009090909000ULL, 0x3300003333333300ULL,
        0x7300007373737300ULL, 0x6700006767676700ULL, 0xf60000f6f6f6f600ULL,
        0xf30000f3f3f3f300ULL, 0x9d00009d9d9d9d00ULL, 0x7f00007f7f7f7f00ULL,
        0xbf0000bfbfbfbf00ULL, 0xe20000e2e2e2e200ULL, 0x5200005252525200ULL,
        0x9b00009b9b9b9b00ULL, 0xd80000d8d8d8d800ULL, 0x2600002626262600ULL,
        0xc80000c8c8c8c800ULL, 0x3700003737373700ULL, 0xc60000c6c6c6c600ULL,
        0x3b00003b3b3b3b00ULL, 0x8100008181818100ULL, 0x9600009696969600ULL,
        0x6f00006f6f6f6f00ULL, 0x4b00004b4b4b4b00ULL, 0x1300001313131300ULL,
        0xbe0000bebebebe00ULL, 0x6300006363636300ULL, 0x2e00002e2e2e2e00ULL,
        0xe90000e9e9e9e900ULL, 0x7900007979797900ULL, 0xa70000a7a7a7a700ULL,
        0x8c00008c8c8c8c00ULL, 0x9f00009f9f9f9f00ULL, 0x6e00006e6e6e6e00ULL,
        0xbc0000bcbcbcbc00ULL, 0x8e00008e8e8e8e00ULL, 0x2900002929292900ULL,
        0xf50000f5f5f5f500ULL, 0xf90000f9f9f9f900ULL, 0xb60000b6b6b6b600ULL,
        0x2f00002f2f2f2f00ULL, 0xfd0000fdfdfdfd00ULL, 0xb40000b4b4b4b400ULL,
        0x5900005959595900ULL, 0x7800007878787800ULL, 0x9800009898989800ULL,
        0x0600000606060600ULL, 0x6a00006a6a6a6a00ULL, 0xe70000e7e7e7e700ULL,
        0x4600004646464600ULL, 0x7100007171717100ULL, 0xba0000babababa00ULL,
        0xd40000d4d4d4d400ULL, 0x2500002525252500ULL, 0xab0000abababab00ULL,
        0x4200004242424200ULL, 0x8800008888888800ULL, 0xa20000a2a2a2a200ULL,
        0x8d00008d8d8d8d00ULL, 0xfa0000fafafafa00ULL, 0x7200007272727200ULL,
        0x0700000707070700ULL, 0xb90000b9b9b9b900ULL, 0x5500005555555500ULL,
        0xf80000f8f8f8f800ULL, 0xee0000eeeeeeee00ULL, 0xac0000acacacac00ULL,
        0x0a00000a0a0a0a00ULL, 0x3600003636363600ULL, 0x4900004949494900ULL,
        0x2a00002a2a2a2a00ULL, 0x6800006868686800ULL, 0x3c00003c3c3c3c00ULL,
        0x3800003838383800ULL, 0xf10000f1f1f1f100ULL, 0xa40000a4a4a4a400ULL,
        0x4000004040404000ULL, 0x2800002828282800ULL, 0xd30000d3d3d3d300ULL,
        0x7b00007b7b7b7b00ULL, 0xbb0000bbbbbbbb00ULL, 0xc90000c9c9c9c900ULL,
        0x4300004343434300ULL, 0xc10000c1c1c1c100ULL, 0x1500001515151500ULL,
        0xe30000e3e3e3e300ULL, 0xad0000adadadad00ULL, 0xf40000f4f4f4f400ULL,
        0x7700007777777700ULL, 0xc70000c7c7c7c700ULL, 0x8000008080808000ULL,
        0x9e00009e9e9e9e00ULL,
};

__visible const u64 camellia_sp22000222[256] = {
        0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL,
        0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL,
        0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL,
        0x0b0b0000000b0b0bULL, 0xaeae000000aeaeaeULL, 0x6a6a0000006a6a6aULL,
        0xd5d5000000d5d5d5ULL, 0x1818000000181818ULL, 0x5d5d0000005d5d5dULL,
        0x8282000000828282ULL, 0x4646000000464646ULL, 0xdfdf000000dfdfdfULL,
        0xd6d6000000d6d6d6ULL, 0x2727000000272727ULL, 0x8a8a0000008a8a8aULL,
        0x3232000000323232ULL, 0x4b4b0000004b4b4bULL, 0x4242000000424242ULL,
        0xdbdb000000dbdbdbULL, 0x1c1c0000001c1c1cULL, 0x9e9e0000009e9e9eULL,
        0x9c9c0000009c9c9cULL, 0x3a3a0000003a3a3aULL, 0xcaca000000cacacaULL,
        0x2525000000252525ULL, 0x7b7b0000007b7b7bULL, 0x0d0d0000000d0d0dULL,
        0x7171000000717171ULL, 0x5f5f0000005f5f5fULL, 0x1f1f0000001f1f1fULL,
        0xf8f8000000f8f8f8ULL, 0xd7d7000000d7d7d7ULL, 0x3e3e0000003e3e3eULL,
        0x9d9d0000009d9d9dULL, 0x7c7c0000007c7c7cULL, 0x6060000000606060ULL,
        0xb9b9000000b9b9b9ULL, 0xbebe000000bebebeULL, 0xbcbc000000bcbcbcULL,
        0x8b8b0000008b8b8bULL, 0x1616000000161616ULL, 0x3434000000343434ULL,
        0x4d4d0000004d4d4dULL, 0xc3c3000000c3c3c3ULL, 0x7272000000727272ULL,
        0x9595000000959595ULL, 0xabab000000abababULL, 0x8e8e0000008e8e8eULL,
        0xbaba000000bababaULL, 0x7a7a0000007a7a7aULL, 0xb3b3000000b3b3b3ULL,
        0x0202000000020202ULL, 0xb4b4000000b4b4b4ULL, 0xadad000000adadadULL,
        0xa2a2000000a2a2a2ULL, 0xacac000000acacacULL, 0xd8d8000000d8d8d8ULL,
        0x9a9a0000009a9a9aULL, 0x1717000000171717ULL, 0x1a1a0000001a1a1aULL,
        0x3535000000353535ULL, 0xcccc000000ccccccULL, 0xf7f7000000f7f7f7ULL,
        0x9999000000999999ULL, 0x6161000000616161ULL, 0x5a5a0000005a5a5aULL,
        0xe8e8000000e8e8e8ULL, 0x2424000000242424ULL, 0x5656000000565656ULL,
        0x4040000000404040ULL, 0xe1e1000000e1e1e1ULL, 0x6363000000636363ULL,
        0x0909000000090909ULL, 0x3333000000333333ULL, 0xbfbf000000bfbfbfULL,
        0x9898000000989898ULL, 0x9797000000979797ULL, 0x8585000000858585ULL,
        0x6868000000686868ULL, 0xfcfc000000fcfcfcULL, 0xecec000000ecececULL,
        0x0a0a0000000a0a0aULL, 0xdada000000dadadaULL, 0x6f6f0000006f6f6fULL,
        0x5353000000535353ULL, 0x6262000000626262ULL, 0xa3a3000000a3a3a3ULL,
        0x2e2e0000002e2e2eULL, 0x0808000000080808ULL, 0xafaf000000afafafULL,
        0x2828000000282828ULL, 0xb0b0000000b0b0b0ULL, 0x7474000000747474ULL,
        0xc2c2000000c2c2c2ULL, 0xbdbd000000bdbdbdULL, 0x3636000000363636ULL,
        0x2222000000222222ULL, 0x3838000000383838ULL, 0x6464000000646464ULL,
        0x1e1e0000001e1e1eULL, 0x3939000000393939ULL, 0x2c2c0000002c2c2cULL,
        0xa6a6000000a6a6a6ULL, 0x3030000000303030ULL, 0xe5e5000000e5e5e5ULL,
        0x4444000000444444ULL, 0xfdfd000000fdfdfdULL, 0x8888000000888888ULL,
        0x9f9f0000009f9f9fULL, 0x6565000000656565ULL, 0x8787000000878787ULL,
        0x6b6b0000006b6b6bULL, 0xf4f4000000f4f4f4ULL, 0x2323000000232323ULL,
        0x4848000000484848ULL, 0x1010000000101010ULL, 0xd1d1000000d1d1d1ULL,
        0x5151000000515151ULL, 0xc0c0000000c0c0c0ULL, 0xf9f9000000f9f9f9ULL,
        0xd2d2000000d2d2d2ULL, 0xa0a0000000a0a0a0ULL, 0x5555000000555555ULL,
        0xa1a1000000a1a1a1ULL, 0x4141000000414141ULL, 0xfafa000000fafafaULL,
        0x4343000000434343ULL, 0x1313000000131313ULL, 0xc4c4000000c4c4c4ULL,
        0x2f2f0000002f2f2fULL, 0xa8a8000000a8a8a8ULL, 0xb6b6000000b6b6b6ULL,
        0x3c3c0000003c3c3cULL, 0x2b2b0000002b2b2bULL, 0xc1c1000000c1c1c1ULL,
        0xffff000000ffffffULL, 0xc8c8000000c8c8c8ULL, 0xa5a5000000a5a5a5ULL,
        0x2020000000202020ULL, 0x8989000000898989ULL, 0x0000000000000000ULL,
        0x9090000000909090ULL, 0x4747000000474747ULL, 0xefef000000efefefULL,
        0xeaea000000eaeaeaULL, 0xb7b7000000b7b7b7ULL, 0x1515000000151515ULL,
        0x0606000000060606ULL, 0xcdcd000000cdcdcdULL, 0xb5b5000000b5b5b5ULL,
        0x1212000000121212ULL, 0x7e7e0000007e7e7eULL, 0xbbbb000000bbbbbbULL,
        0x2929000000292929ULL, 0x0f0f0000000f0f0fULL, 0xb8b8000000b8b8b8ULL,
        0x0707000000070707ULL, 0x0404000000040404ULL, 0x9b9b0000009b9b9bULL,
        0x9494000000949494ULL, 0x2121000000212121ULL, 0x6666000000666666ULL,
        0xe6e6000000e6e6e6ULL, 0xcece000000cececeULL, 0xeded000000edededULL,
        0xe7e7000000e7e7e7ULL, 0x3b3b0000003b3b3bULL, 0xfefe000000fefefeULL,
        0x7f7f0000007f7f7fULL, 0xc5c5000000c5c5c5ULL, 0xa4a4000000a4a4a4ULL,
        0x3737000000373737ULL, 0xb1b1000000b1b1b1ULL, 0x4c4c0000004c4c4cULL,
        0x9191000000919191ULL, 0x6e6e0000006e6e6eULL, 0x8d8d0000008d8d8dULL,
        0x7676000000767676ULL, 0x0303000000030303ULL, 0x2d2d0000002d2d2dULL,
        0xdede000000dededeULL, 0x9696000000969696ULL, 0x2626000000262626ULL,
        0x7d7d0000007d7d7dULL, 0xc6c6000000c6c6c6ULL, 0x5c5c0000005c5c5cULL,
        0xd3d3000000d3d3d3ULL, 0xf2f2000000f2f2f2ULL, 0x4f4f0000004f4f4fULL,
        0x1919000000191919ULL, 0x3f3f0000003f3f3fULL, 0xdcdc000000dcdcdcULL,
        0x7979000000797979ULL, 0x1d1d0000001d1d1dULL, 0x5252000000525252ULL,
        0xebeb000000ebebebULL, 0xf3f3000000f3f3f3ULL, 0x6d6d0000006d6d6dULL,
        0x5e5e0000005e5e5eULL, 0xfbfb000000fbfbfbULL, 0x6969000000696969ULL,
        0xb2b2000000b2b2b2ULL, 0xf0f0000000f0f0f0ULL, 0x3131000000313131ULL,
        0x0c0c0000000c0c0cULL, 0xd4d4000000d4d4d4ULL, 0xcfcf000000cfcfcfULL,
        0x8c8c0000008c8c8cULL, 0xe2e2000000e2e2e2ULL, 0x7575000000757575ULL,
        0xa9a9000000a9a9a9ULL, 0x4a4a0000004a4a4aULL, 0x5757000000575757ULL,
        0x8484000000848484ULL, 0x1111000000111111ULL, 0x4545000000454545ULL,
        0x1b1b0000001b1b1bULL, 0xf5f5000000f5f5f5ULL, 0xe4e4000000e4e4e4ULL,
        0x0e0e0000000e0e0eULL, 0x7373000000737373ULL, 0xaaaa000000aaaaaaULL,
        0xf1f1000000f1f1f1ULL, 0xdddd000000ddddddULL, 0x5959000000595959ULL,
        0x1414000000141414ULL, 0x6c6c0000006c6c6cULL, 0x9292000000929292ULL,
        0x5454000000545454ULL, 0xd0d0000000d0d0d0ULL, 0x7878000000787878ULL,
        0x7070000000707070ULL, 0xe3e3000000e3e3e3ULL, 0x4949000000494949ULL,
        0x8080000000808080ULL, 0x5050000000505050ULL, 0xa7a7000000a7a7a7ULL,
        0xf6f6000000f6f6f6ULL, 0x7777000000777777ULL, 0x9393000000939393ULL,
        0x8686000000868686ULL, 0x8383000000838383ULL, 0x2a2a0000002a2a2aULL,
        0xc7c7000000c7c7c7ULL, 0x5b5b0000005b5b5bULL, 0xe9e9000000e9e9e9ULL,
        0xeeee000000eeeeeeULL, 0x8f8f0000008f8f8fULL, 0x0101000000010101ULL,
        0x3d3d0000003d3d3dULL,
};

__visible const u64 camellia_sp03303033[256] = {
        0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL,
        0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL,
        0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL,
        0x00c2c200c200c2c2ULL, 0x00abab00ab00ababULL, 0x009a9a009a009a9aULL,
        0x0075750075007575ULL, 0x0006060006000606ULL, 0x0057570057005757ULL,
        0x00a0a000a000a0a0ULL, 0x0091910091009191ULL, 0x00f7f700f700f7f7ULL,
        0x00b5b500b500b5b5ULL, 0x00c9c900c900c9c9ULL, 0x00a2a200a200a2a2ULL,
        0x008c8c008c008c8cULL, 0x00d2d200d200d2d2ULL, 0x0090900090009090ULL,
        0x00f6f600f600f6f6ULL, 0x0007070007000707ULL, 0x00a7a700a700a7a7ULL,
        0x0027270027002727ULL, 0x008e8e008e008e8eULL, 0x00b2b200b200b2b2ULL,
        0x0049490049004949ULL, 0x00dede00de00dedeULL, 0x0043430043004343ULL,
        0x005c5c005c005c5cULL, 0x00d7d700d700d7d7ULL, 0x00c7c700c700c7c7ULL,
        0x003e3e003e003e3eULL, 0x00f5f500f500f5f5ULL, 0x008f8f008f008f8fULL,
        0x0067670067006767ULL, 0x001f1f001f001f1fULL, 0x0018180018001818ULL,
        0x006e6e006e006e6eULL, 0x00afaf00af00afafULL, 0x002f2f002f002f2fULL,
        0x00e2e200e200e2e2ULL, 0x0085850085008585ULL, 0x000d0d000d000d0dULL,
        0x0053530053005353ULL, 0x00f0f000f000f0f0ULL, 0x009c9c009c009c9cULL,
        0x0065650065006565ULL, 0x00eaea00ea00eaeaULL, 0x00a3a300a300a3a3ULL,
        0x00aeae00ae00aeaeULL, 0x009e9e009e009e9eULL, 0x00ecec00ec00ececULL,
        0x0080800080008080ULL, 0x002d2d002d002d2dULL, 0x006b6b006b006b6bULL,
        0x00a8a800a800a8a8ULL, 0x002b2b002b002b2bULL, 0x0036360036003636ULL,
        0x00a6a600a600a6a6ULL, 0x00c5c500c500c5c5ULL, 0x0086860086008686ULL,
        0x004d4d004d004d4dULL, 0x0033330033003333ULL, 0x00fdfd00fd00fdfdULL,
        0x0066660066006666ULL, 0x0058580058005858ULL, 0x0096960096009696ULL,
        0x003a3a003a003a3aULL, 0x0009090009000909ULL, 0x0095950095009595ULL,
        0x0010100010001010ULL, 0x0078780078007878ULL, 0x00d8d800d800d8d8ULL,
        0x0042420042004242ULL, 0x00cccc00cc00ccccULL, 0x00efef00ef00efefULL,
        0x0026260026002626ULL, 0x00e5e500e500e5e5ULL, 0x0061610061006161ULL,
        0x001a1a001a001a1aULL, 0x003f3f003f003f3fULL, 0x003b3b003b003b3bULL,
        0x0082820082008282ULL, 0x00b6b600b600b6b6ULL, 0x00dbdb00db00dbdbULL,
        0x00d4d400d400d4d4ULL, 0x0098980098009898ULL, 0x00e8e800e800e8e8ULL,
        0x008b8b008b008b8bULL, 0x0002020002000202ULL, 0x00ebeb00eb00ebebULL,
        0x000a0a000a000a0aULL, 0x002c2c002c002c2cULL, 0x001d1d001d001d1dULL,
        0x00b0b000b000b0b0ULL, 0x006f6f006f006f6fULL, 0x008d8d008d008d8dULL,
        0x0088880088008888ULL, 0x000e0e000e000e0eULL, 0x0019190019001919ULL,
        0x0087870087008787ULL, 0x004e4e004e004e4eULL, 0x000b0b000b000b0bULL,
        0x00a9a900a900a9a9ULL, 0x000c0c000c000c0cULL, 0x0079790079007979ULL,
        0x0011110011001111ULL, 0x007f7f007f007f7fULL, 0x0022220022002222ULL,
        0x00e7e700e700e7e7ULL, 0x0059590059005959ULL, 0x00e1e100e100e1e1ULL,
        0x00dada00da00dadaULL, 0x003d3d003d003d3dULL, 0x00c8c800c800c8c8ULL,
        0x0012120012001212ULL, 0x0004040004000404ULL, 0x0074740074007474ULL,
        0x0054540054005454ULL, 0x0030300030003030ULL, 0x007e7e007e007e7eULL,
        0x00b4b400b400b4b4ULL, 0x0028280028002828ULL, 0x0055550055005555ULL,
        0x0068680068006868ULL, 0x0050500050005050ULL, 0x00bebe00be00bebeULL,
        0x00d0d000d000d0d0ULL, 0x00c4c400c400c4c4ULL, 0x0031310031003131ULL,
        0x00cbcb00cb00cbcbULL, 0x002a2a002a002a2aULL, 0x00adad00ad00adadULL,
        0x000f0f000f000f0fULL, 0x00caca00ca00cacaULL, 0x0070700070007070ULL,
        0x00ffff00ff00ffffULL, 0x0032320032003232ULL, 0x0069690069006969ULL,
        0x0008080008000808ULL, 0x0062620062006262ULL, 0x0000000000000000ULL,
        0x0024240024002424ULL, 0x00d1d100d100d1d1ULL, 0x00fbfb00fb00fbfbULL,
        0x00baba00ba00babaULL, 0x00eded00ed00ededULL, 0x0045450045004545ULL,
        0x0081810081008181ULL, 0x0073730073007373ULL, 0x006d6d006d006d6dULL,
        0x0084840084008484ULL, 0x009f9f009f009f9fULL, 0x00eeee00ee00eeeeULL,
        0x004a4a004a004a4aULL, 0x00c3c300c300c3c3ULL, 0x002e2e002e002e2eULL,
        0x00c1c100c100c1c1ULL, 0x0001010001000101ULL, 0x00e6e600e600e6e6ULL,
        0x0025250025002525ULL, 0x0048480048004848ULL, 0x0099990099009999ULL,
        0x00b9b900b900b9b9ULL, 0x00b3b300b300b3b3ULL, 0x007b7b007b007b7bULL,
        0x00f9f900f900f9f9ULL, 0x00cece00ce00ceceULL, 0x00bfbf00bf00bfbfULL,
        0x00dfdf00df00dfdfULL, 0x0071710071007171ULL, 0x0029290029002929ULL,
        0x00cdcd00cd00cdcdULL, 0x006c6c006c006c6cULL, 0x0013130013001313ULL,
        0x0064640064006464ULL, 0x009b9b009b009b9bULL, 0x0063630063006363ULL,
        0x009d9d009d009d9dULL, 0x00c0c000c000c0c0ULL, 0x004b4b004b004b4bULL,
        0x00b7b700b700b7b7ULL, 0x00a5a500a500a5a5ULL, 0x0089890089008989ULL,
        0x005f5f005f005f5fULL, 0x00b1b100b100b1b1ULL, 0x0017170017001717ULL,
        0x00f4f400f400f4f4ULL, 0x00bcbc00bc00bcbcULL, 0x00d3d300d300d3d3ULL,
        0x0046460046004646ULL, 0x00cfcf00cf00cfcfULL, 0x0037370037003737ULL,
        0x005e5e005e005e5eULL, 0x0047470047004747ULL, 0x0094940094009494ULL,
        0x00fafa00fa00fafaULL, 0x00fcfc00fc00fcfcULL, 0x005b5b005b005b5bULL,
        0x0097970097009797ULL, 0x00fefe00fe00fefeULL, 0x005a5a005a005a5aULL,
        0x00acac00ac00acacULL, 0x003c3c003c003c3cULL, 0x004c4c004c004c4cULL,
        0x0003030003000303ULL, 0x0035350035003535ULL, 0x00f3f300f300f3f3ULL,
        0x0023230023002323ULL, 0x00b8b800b800b8b8ULL, 0x005d5d005d005d5dULL,
        0x006a6a006a006a6aULL, 0x0092920092009292ULL, 0x00d5d500d500d5d5ULL,
        0x0021210021002121ULL, 0x0044440044004444ULL, 0x0051510051005151ULL,
        0x00c6c600c600c6c6ULL, 0x007d7d007d007d7dULL, 0x0039390039003939ULL,
        0x0083830083008383ULL, 0x00dcdc00dc00dcdcULL, 0x00aaaa00aa00aaaaULL,
        0x007c7c007c007c7cULL, 0x0077770077007777ULL, 0x0056560056005656ULL,
        0x0005050005000505ULL, 0x001b1b001b001b1bULL, 0x00a4a400a400a4a4ULL,
        0x0015150015001515ULL, 0x0034340034003434ULL, 0x001e1e001e001e1eULL,
        0x001c1c001c001c1cULL, 0x00f8f800f800f8f8ULL, 0x0052520052005252ULL,
        0x0020200020002020ULL, 0x0014140014001414ULL, 0x00e9e900e900e9e9ULL,
        0x00bdbd00bd00bdbdULL, 0x00dddd00dd00ddddULL, 0x00e4e400e400e4e4ULL,
        0x00a1a100a100a1a1ULL, 0x00e0e000e000e0e0ULL, 0x008a8a008a008a8aULL,
        0x00f1f100f100f1f1ULL, 0x00d6d600d600d6d6ULL, 0x007a7a007a007a7aULL,
        0x00bbbb00bb00bbbbULL, 0x00e3e300e300e3e3ULL, 0x0040400040004040ULL,
        0x004f4f004f004f4fULL,
};

__visible const u64 camellia_sp00444404[256] = {
        0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL,
        0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL,
        0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL,
        0x00006b6b6b6b006bULL, 0x0000454545450045ULL, 0x0000a5a5a5a500a5ULL,
        0x0000edededed00edULL, 0x00004f4f4f4f004fULL, 0x00001d1d1d1d001dULL,
        0x0000929292920092ULL, 0x0000868686860086ULL, 0x0000afafafaf00afULL,
        0x00007c7c7c7c007cULL, 0x00001f1f1f1f001fULL, 0x00003e3e3e3e003eULL,
        0x0000dcdcdcdc00dcULL, 0x00005e5e5e5e005eULL, 0x00000b0b0b0b000bULL,
        0x0000a6a6a6a600a6ULL, 0x0000393939390039ULL, 0x0000d5d5d5d500d5ULL,
        0x00005d5d5d5d005dULL, 0x0000d9d9d9d900d9ULL, 0x00005a5a5a5a005aULL,
        0x0000515151510051ULL, 0x00006c6c6c6c006cULL, 0x00008b8b8b8b008bULL,
        0x00009a9a9a9a009aULL, 0x0000fbfbfbfb00fbULL, 0x0000b0b0b0b000b0ULL,
        0x0000747474740074ULL, 0x00002b2b2b2b002bULL, 0x0000f0f0f0f000f0ULL,
        0x0000848484840084ULL, 0x0000dfdfdfdf00dfULL, 0x0000cbcbcbcb00cbULL,
        0x0000343434340034ULL, 0x0000767676760076ULL, 0x00006d6d6d6d006dULL,
        0x0000a9a9a9a900a9ULL, 0x0000d1d1d1d100d1ULL, 0x0000040404040004ULL,
        0x0000141414140014ULL, 0x00003a3a3a3a003aULL, 0x0000dededede00deULL,
        0x0000111111110011ULL, 0x0000323232320032ULL, 0x00009c9c9c9c009cULL,
        0x0000535353530053ULL, 0x0000f2f2f2f200f2ULL, 0x0000fefefefe00feULL,
        0x0000cfcfcfcf00cfULL, 0x0000c3c3c3c300c3ULL, 0x00007a7a7a7a007aULL,
        0x0000242424240024ULL, 0x0000e8e8e8e800e8ULL, 0x0000606060600060ULL,
        0x0000696969690069ULL, 0x0000aaaaaaaa00aaULL, 0x0000a0a0a0a000a0ULL,
        0x0000a1a1a1a100a1ULL, 0x0000626262620062ULL, 0x0000545454540054ULL,
        0x00001e1e1e1e001eULL, 0x0000e0e0e0e000e0ULL, 0x0000646464640064ULL,
        0x0000101010100010ULL, 0x0000000000000000ULL, 0x0000a3a3a3a300a3ULL,
        0x0000757575750075ULL, 0x00008a8a8a8a008aULL, 0x0000e6e6e6e600e6ULL,
        0x0000090909090009ULL, 0x0000dddddddd00ddULL, 0x0000878787870087ULL,
        0x0000838383830083ULL, 0x0000cdcdcdcd00cdULL, 0x0000909090900090ULL,
        0x0000737373730073ULL, 0x0000f6f6f6f600f6ULL, 0x00009d9d9d9d009dULL,
        0x0000bfbfbfbf00bfULL, 0x0000525252520052ULL, 0x0000d8d8d8d800d8ULL,
        0x0000c8c8c8c800c8ULL, 0x0000c6c6c6c600c6ULL, 0x0000818181810081ULL,
        0x00006f6f6f6f006fULL, 0x0000131313130013ULL, 0x0000636363630063ULL,
        0x0000e9e9e9e900e9ULL, 0x0000a7a7a7a700a7ULL, 0x00009f9f9f9f009fULL,
        0x0000bcbcbcbc00bcULL, 0x0000292929290029ULL, 0x0000f9f9f9f900f9ULL,
        0x00002f2f2f2f002fULL, 0x0000b4b4b4b400b4ULL, 0x0000787878780078ULL,
        0x0000060606060006ULL, 0x0000e7e7e7e700e7ULL, 0x0000717171710071ULL,
        0x0000d4d4d4d400d4ULL, 0x0000abababab00abULL, 0x0000888888880088ULL,
        0x00008d8d8d8d008dULL, 0x0000727272720072ULL, 0x0000b9b9b9b900b9ULL,
        0x0000f8f8f8f800f8ULL, 0x0000acacacac00acULL, 0x0000363636360036ULL,
        0x00002a2a2a2a002aULL, 0x00003c3c3c3c003cULL, 0x0000f1f1f1f100f1ULL,
        0x0000404040400040ULL, 0x0000d3d3d3d300d3ULL, 0x0000bbbbbbbb00bbULL,
        0x0000434343430043ULL, 0x0000151515150015ULL, 0x0000adadadad00adULL,
        0x0000777777770077ULL, 0x0000808080800080ULL, 0x0000828282820082ULL,
        0x0000ecececec00ecULL, 0x0000272727270027ULL, 0x0000e5e5e5e500e5ULL,
        0x0000858585850085ULL, 0x0000353535350035ULL, 0x00000c0c0c0c000cULL,
        0x0000414141410041ULL, 0x0000efefefef00efULL, 0x0000939393930093ULL,
        0x0000191919190019ULL, 0x0000212121210021ULL, 0x00000e0e0e0e000eULL,
        0x00004e4e4e4e004eULL, 0x0000656565650065ULL, 0x0000bdbdbdbd00bdULL,
        0x0000b8b8b8b800b8ULL, 0x00008f8f8f8f008fULL, 0x0000ebebebeb00ebULL,
        0x0000cececece00ceULL, 0x0000303030300030ULL, 0x00005f5f5f5f005fULL,
        0x0000c5c5c5c500c5ULL, 0x00001a1a1a1a001aULL, 0x0000e1e1e1e100e1ULL,
        0x0000cacacaca00caULL, 0x0000474747470047ULL, 0x00003d3d3d3d003dULL,
        0x0000010101010001ULL, 0x0000d6d6d6d600d6ULL, 0x0000565656560056ULL,
        0x00004d4d4d4d004dULL, 0x00000d0d0d0d000dULL, 0x0000666666660066ULL,
        0x0000cccccccc00ccULL, 0x00002d2d2d2d002dULL, 0x0000121212120012ULL,
        0x0000202020200020ULL, 0x0000b1b1b1b100b1ULL, 0x0000999999990099ULL,
        0x00004c4c4c4c004cULL, 0x0000c2c2c2c200c2ULL, 0x00007e7e7e7e007eULL,
        0x0000050505050005ULL, 0x0000b7b7b7b700b7ULL, 0x0000313131310031ULL,
        0x0000171717170017ULL, 0x0000d7d7d7d700d7ULL, 0x0000585858580058ULL,
        0x0000616161610061ULL, 0x00001b1b1b1b001bULL, 0x00001c1c1c1c001cULL,
        0x00000f0f0f0f000fULL, 0x0000161616160016ULL, 0x0000181818180018ULL,
        0x0000222222220022ULL, 0x0000444444440044ULL, 0x0000b2b2b2b200b2ULL,
        0x0000b5b5b5b500b5ULL, 0x0000919191910091ULL, 0x0000080808080008ULL,
        0x0000a8a8a8a800a8ULL, 0x0000fcfcfcfc00fcULL, 0x0000505050500050ULL,
        0x0000d0d0d0d000d0ULL, 0x00007d7d7d7d007dULL, 0x0000898989890089ULL,
        0x0000979797970097ULL, 0x00005b5b5b5b005bULL, 0x0000959595950095ULL,
        0x0000ffffffff00ffULL, 0x0000d2d2d2d200d2ULL, 0x0000c4c4c4c400c4ULL,
        0x0000484848480048ULL, 0x0000f7f7f7f700f7ULL, 0x0000dbdbdbdb00dbULL,
        0x0000030303030003ULL, 0x0000dadadada00daULL, 0x00003f3f3f3f003fULL,
        0x0000949494940094ULL, 0x00005c5c5c5c005cULL, 0x0000020202020002ULL,
        0x00004a4a4a4a004aULL, 0x0000333333330033ULL, 0x0000676767670067ULL,
        0x0000f3f3f3f300f3ULL, 0x00007f7f7f7f007fULL, 0x0000e2e2e2e200e2ULL,
        0x00009b9b9b9b009bULL, 0x0000262626260026ULL, 0x0000373737370037ULL,
        0x00003b3b3b3b003bULL, 0x0000969696960096ULL, 0x00004b4b4b4b004bULL,
        0x0000bebebebe00beULL, 0x00002e2e2e2e002eULL, 0x0000797979790079ULL,
        0x00008c8c8c8c008cULL, 0x00006e6e6e6e006eULL, 0x00008e8e8e8e008eULL,
        0x0000f5f5f5f500f5ULL, 0x0000b6b6b6b600b6ULL, 0x0000fdfdfdfd00fdULL,
        0x0000595959590059ULL, 0x0000989898980098ULL, 0x00006a6a6a6a006aULL,
        0x0000464646460046ULL, 0x0000babababa00baULL, 0x0000252525250025ULL,
        0x0000424242420042ULL, 0x0000a2a2a2a200a2ULL, 0x0000fafafafa00faULL,
        0x0000070707070007ULL, 0x0000555555550055ULL, 0x0000eeeeeeee00eeULL,
        0x00000a0a0a0a000aULL, 0x0000494949490049ULL, 0x0000686868680068ULL,
        0x0000383838380038ULL, 0x0000a4a4a4a400a4ULL, 0x0000282828280028ULL,
        0x00007b7b7b7b007bULL, 0x0000c9c9c9c900c9ULL, 0x0000c1c1c1c100c1ULL,
        0x0000e3e3e3e300e3ULL, 0x0000f4f4f4f400f4ULL, 0x0000c7c7c7c700c7ULL,
        0x00009e9e9e9e009eULL,
};

__visible const u64 camellia_sp02220222[256] = {
        0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL,
        0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL,
        0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL,
        0x000b0b0b000b0b0bULL, 0x00aeaeae00aeaeaeULL, 0x006a6a6a006a6a6aULL,
        0x00d5d5d500d5d5d5ULL, 0x0018181800181818ULL, 0x005d5d5d005d5d5dULL,
        0x0082828200828282ULL, 0x0046464600464646ULL, 0x00dfdfdf00dfdfdfULL,
        0x00d6d6d600d6d6d6ULL, 0x0027272700272727ULL, 0x008a8a8a008a8a8aULL,
        0x0032323200323232ULL, 0x004b4b4b004b4b4bULL, 0x0042424200424242ULL,
        0x00dbdbdb00dbdbdbULL, 0x001c1c1c001c1c1cULL, 0x009e9e9e009e9e9eULL,
        0x009c9c9c009c9c9cULL, 0x003a3a3a003a3a3aULL, 0x00cacaca00cacacaULL,
        0x0025252500252525ULL, 0x007b7b7b007b7b7bULL, 0x000d0d0d000d0d0dULL,
        0x0071717100717171ULL, 0x005f5f5f005f5f5fULL, 0x001f1f1f001f1f1fULL,
        0x00f8f8f800f8f8f8ULL, 0x00d7d7d700d7d7d7ULL, 0x003e3e3e003e3e3eULL,
        0x009d9d9d009d9d9dULL, 0x007c7c7c007c7c7cULL, 0x0060606000606060ULL,
        0x00b9b9b900b9b9b9ULL, 0x00bebebe00bebebeULL, 0x00bcbcbc00bcbcbcULL,
        0x008b8b8b008b8b8bULL, 0x0016161600161616ULL, 0x0034343400343434ULL,
        0x004d4d4d004d4d4dULL, 0x00c3c3c300c3c3c3ULL, 0x0072727200727272ULL,
        0x0095959500959595ULL, 0x00ababab00abababULL, 0x008e8e8e008e8e8eULL,
        0x00bababa00bababaULL, 0x007a7a7a007a7a7aULL, 0x00b3b3b300b3b3b3ULL,
        0x0002020200020202ULL, 0x00b4b4b400b4b4b4ULL, 0x00adadad00adadadULL,
        0x00a2a2a200a2a2a2ULL, 0x00acacac00acacacULL, 0x00d8d8d800d8d8d8ULL,
        0x009a9a9a009a9a9aULL, 0x0017171700171717ULL, 0x001a1a1a001a1a1aULL,
        0x0035353500353535ULL, 0x00cccccc00ccccccULL, 0x00f7f7f700f7f7f7ULL,
        0x0099999900999999ULL, 0x0061616100616161ULL, 0x005a5a5a005a5a5aULL,
        0x00e8e8e800e8e8e8ULL, 0x0024242400242424ULL, 0x0056565600565656ULL,
        0x0040404000404040ULL, 0x00e1e1e100e1e1e1ULL, 0x0063636300636363ULL,
        0x0009090900090909ULL, 0x0033333300333333ULL, 0x00bfbfbf00bfbfbfULL,
        0x0098989800989898ULL, 0x0097979700979797ULL, 0x0085858500858585ULL,
        0x0068686800686868ULL, 0x00fcfcfc00fcfcfcULL, 0x00ececec00ecececULL,
        0x000a0a0a000a0a0aULL, 0x00dadada00dadadaULL, 0x006f6f6f006f6f6fULL,
        0x0053535300535353ULL, 0x0062626200626262ULL, 0x00a3a3a300a3a3a3ULL,
        0x002e2e2e002e2e2eULL, 0x0008080800080808ULL, 0x00afafaf00afafafULL,
        0x0028282800282828ULL, 0x00b0b0b000b0b0b0ULL, 0x0074747400747474ULL,
        0x00c2c2c200c2c2c2ULL, 0x00bdbdbd00bdbdbdULL, 0x0036363600363636ULL,
        0x0022222200222222ULL, 0x0038383800383838ULL, 0x0064646400646464ULL,
        0x001e1e1e001e1e1eULL, 0x0039393900393939ULL, 0x002c2c2c002c2c2cULL,
        0x00a6a6a600a6a6a6ULL, 0x0030303000303030ULL, 0x00e5e5e500e5e5e5ULL,
        0x0044444400444444ULL, 0x00fdfdfd00fdfdfdULL, 0x0088888800888888ULL,
        0x009f9f9f009f9f9fULL, 0x0065656500656565ULL, 0x0087878700878787ULL,
        0x006b6b6b006b6b6bULL, 0x00f4f4f400f4f4f4ULL, 0x0023232300232323ULL,
        0x0048484800484848ULL, 0x0010101000101010ULL, 0x00d1d1d100d1d1d1ULL,
        0x0051515100515151ULL, 0x00c0c0c000c0c0c0ULL, 0x00f9f9f900f9f9f9ULL,
        0x00d2d2d200d2d2d2ULL, 0x00a0a0a000a0a0a0ULL, 0x0055555500555555ULL,
        0x00a1a1a100a1a1a1ULL, 0x0041414100414141ULL, 0x00fafafa00fafafaULL,
        0x0043434300434343ULL, 0x0013131300131313ULL, 0x00c4c4c400c4c4c4ULL,
        0x002f2f2f002f2f2fULL, 0x00a8a8a800a8a8a8ULL, 0x00b6b6b600b6b6b6ULL,
        0x003c3c3c003c3c3cULL, 0x002b2b2b002b2b2bULL, 0x00c1c1c100c1c1c1ULL,
        0x00ffffff00ffffffULL, 0x00c8c8c800c8c8c8ULL, 0x00a5a5a500a5a5a5ULL,
        0x0020202000202020ULL, 0x0089898900898989ULL, 0x0000000000000000ULL,
        0x0090909000909090ULL, 0x0047474700474747ULL, 0x00efefef00efefefULL,
        0x00eaeaea00eaeaeaULL, 0x00b7b7b700b7b7b7ULL, 0x0015151500151515ULL,
        0x0006060600060606ULL, 0x00cdcdcd00cdcdcdULL, 0x00b5b5b500b5b5b5ULL,
        0x0012121200121212ULL, 0x007e7e7e007e7e7eULL, 0x00bbbbbb00bbbbbbULL,
        0x0029292900292929ULL, 0x000f0f0f000f0f0fULL, 0x00b8b8b800b8b8b8ULL,
        0x0007070700070707ULL, 0x0004040400040404ULL, 0x009b9b9b009b9b9bULL,
        0x0094949400949494ULL, 0x0021212100212121ULL, 0x0066666600666666ULL,
        0x00e6e6e600e6e6e6ULL, 0x00cecece00cececeULL, 0x00ededed00edededULL,
        0x00e7e7e700e7e7e7ULL, 0x003b3b3b003b3b3bULL, 0x00fefefe00fefefeULL,
        0x007f7f7f007f7f7fULL, 0x00c5c5c500c5c5c5ULL, 0x00a4a4a400a4a4a4ULL,
        0x0037373700373737ULL, 0x00b1b1b100b1b1b1ULL, 0x004c4c4c004c4c4cULL,
        0x0091919100919191ULL, 0x006e6e6e006e6e6eULL, 0x008d8d8d008d8d8dULL,
        0x0076767600767676ULL, 0x0003030300030303ULL, 0x002d2d2d002d2d2dULL,
        0x00dedede00dededeULL, 0x0096969600969696ULL, 0x0026262600262626ULL,
        0x007d7d7d007d7d7dULL, 0x00c6c6c600c6c6c6ULL, 0x005c5c5c005c5c5cULL,
        0x00d3d3d300d3d3d3ULL, 0x00f2f2f200f2f2f2ULL, 0x004f4f4f004f4f4fULL,
        0x0019191900191919ULL, 0x003f3f3f003f3f3fULL, 0x00dcdcdc00dcdcdcULL,
        0x0079797900797979ULL, 0x001d1d1d001d1d1dULL, 0x0052525200525252ULL,
        0x00ebebeb00ebebebULL, 0x00f3f3f300f3f3f3ULL, 0x006d6d6d006d6d6dULL,
        0x005e5e5e005e5e5eULL, 0x00fbfbfb00fbfbfbULL, 0x0069696900696969ULL,
        0x00b2b2b200b2b2b2ULL, 0x00f0f0f000f0f0f0ULL, 0x0031313100313131ULL,
        0x000c0c0c000c0c0cULL, 0x00d4d4d400d4d4d4ULL, 0x00cfcfcf00cfcfcfULL,
        0x008c8c8c008c8c8cULL, 0x00e2e2e200e2e2e2ULL, 0x0075757500757575ULL,
        0x00a9a9a900a9a9a9ULL, 0x004a4a4a004a4a4aULL, 0x0057575700575757ULL,
        0x0084848400848484ULL, 0x0011111100111111ULL, 0x0045454500454545ULL,
        0x001b1b1b001b1b1bULL, 0x00f5f5f500f5f5f5ULL, 0x00e4e4e400e4e4e4ULL,
        0x000e0e0e000e0e0eULL, 0x0073737300737373ULL, 0x00aaaaaa00aaaaaaULL,
        0x00f1f1f100f1f1f1ULL, 0x00dddddd00ddddddULL, 0x0059595900595959ULL,
        0x0014141400141414ULL, 0x006c6c6c006c6c6cULL, 0x0092929200929292ULL,
        0x0054545400545454ULL, 0x00d0d0d000d0d0d0ULL, 0x0078787800787878ULL,
        0x0070707000707070ULL, 0x00e3e3e300e3e3e3ULL, 0x0049494900494949ULL,
        0x0080808000808080ULL, 0x0050505000505050ULL, 0x00a7a7a700a7a7a7ULL,
        0x00f6f6f600f6f6f6ULL, 0x0077777700777777ULL, 0x0093939300939393ULL,
        0x0086868600868686ULL, 0x0083838300838383ULL, 0x002a2a2a002a2a2aULL,
        0x00c7c7c700c7c7c7ULL, 0x005b5b5b005b5b5bULL, 0x00e9e9e900e9e9e9ULL,
        0x00eeeeee00eeeeeeULL, 0x008f8f8f008f8f8fULL, 0x0001010100010101ULL,
        0x003d3d3d003d3d3dULL,
};

__visible const u64 camellia_sp30333033[256] = {
        0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL,
        0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL,
        0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL,
        0xc200c2c2c200c2c2ULL, 0xab00ababab00ababULL, 0x9a009a9a9a009a9aULL,
        0x7500757575007575ULL, 0x0600060606000606ULL, 0x5700575757005757ULL,
        0xa000a0a0a000a0a0ULL, 0x9100919191009191ULL, 0xf700f7f7f700f7f7ULL,
        0xb500b5b5b500b5b5ULL, 0xc900c9c9c900c9c9ULL, 0xa200a2a2a200a2a2ULL,
        0x8c008c8c8c008c8cULL, 0xd200d2d2d200d2d2ULL, 0x9000909090009090ULL,
        0xf600f6f6f600f6f6ULL, 0x0700070707000707ULL, 0xa700a7a7a700a7a7ULL,
        0x2700272727002727ULL, 0x8e008e8e8e008e8eULL, 0xb200b2b2b200b2b2ULL,
        0x4900494949004949ULL, 0xde00dedede00dedeULL, 0x4300434343004343ULL,
        0x5c005c5c5c005c5cULL, 0xd700d7d7d700d7d7ULL, 0xc700c7c7c700c7c7ULL,
        0x3e003e3e3e003e3eULL, 0xf500f5f5f500f5f5ULL, 0x8f008f8f8f008f8fULL,
        0x6700676767006767ULL, 0x1f001f1f1f001f1fULL, 0x1800181818001818ULL,
        0x6e006e6e6e006e6eULL, 0xaf00afafaf00afafULL, 0x2f002f2f2f002f2fULL,
        0xe200e2e2e200e2e2ULL, 0x8500858585008585ULL, 0x0d000d0d0d000d0dULL,
        0x5300535353005353ULL, 0xf000f0f0f000f0f0ULL, 0x9c009c9c9c009c9cULL,
        0x6500656565006565ULL, 0xea00eaeaea00eaeaULL, 0xa300a3a3a300a3a3ULL,
        0xae00aeaeae00aeaeULL, 0x9e009e9e9e009e9eULL, 0xec00ececec00ececULL,
        0x8000808080008080ULL, 0x2d002d2d2d002d2dULL, 0x6b006b6b6b006b6bULL,
        0xa800a8a8a800a8a8ULL, 0x2b002b2b2b002b2bULL, 0x3600363636003636ULL,
        0xa600a6a6a600a6a6ULL, 0xc500c5c5c500c5c5ULL, 0x8600868686008686ULL,
        0x4d004d4d4d004d4dULL, 0x3300333333003333ULL, 0xfd00fdfdfd00fdfdULL,
        0x6600666666006666ULL, 0x5800585858005858ULL, 0x9600969696009696ULL,
        0x3a003a3a3a003a3aULL, 0x0900090909000909ULL, 0x9500959595009595ULL,
        0x1000101010001010ULL, 0x7800787878007878ULL, 0xd800d8d8d800d8d8ULL,
        0x4200424242004242ULL, 0xcc00cccccc00ccccULL, 0xef00efefef00efefULL,
        0x2600262626002626ULL, 0xe500e5e5e500e5e5ULL, 0x6100616161006161ULL,
        0x1a001a1a1a001a1aULL, 0x3f003f3f3f003f3fULL, 0x3b003b3b3b003b3bULL,
        0x8200828282008282ULL, 0xb600b6b6b600b6b6ULL, 0xdb00dbdbdb00dbdbULL,
        0xd400d4d4d400d4d4ULL, 0x9800989898009898ULL, 0xe800e8e8e800e8e8ULL,
        0x8b008b8b8b008b8bULL, 0x0200020202000202ULL, 0xeb00ebebeb00ebebULL,
        0x0a000a0a0a000a0aULL, 0x2c002c2c2c002c2cULL, 0x1d001d1d1d001d1dULL,
        0xb000b0b0b000b0b0ULL, 0x6f006f6f6f006f6fULL, 0x8d008d8d8d008d8dULL,
        0x8800888888008888ULL, 0x0e000e0e0e000e0eULL, 0x1900191919001919ULL,
        0x8700878787008787ULL, 0x4e004e4e4e004e4eULL, 0x0b000b0b0b000b0bULL,
        0xa900a9a9a900a9a9ULL, 0x0c000c0c0c000c0cULL, 0x7900797979007979ULL,
        0x1100111111001111ULL, 0x7f007f7f7f007f7fULL, 0x2200222222002222ULL,
        0xe700e7e7e700e7e7ULL, 0x5900595959005959ULL, 0xe100e1e1e100e1e1ULL,
        0xda00dadada00dadaULL, 0x3d003d3d3d003d3dULL, 0xc800c8c8c800c8c8ULL,
        0x1200121212001212ULL, 0x0400040404000404ULL, 0x7400747474007474ULL,
        0x5400545454005454ULL, 0x3000303030003030ULL, 0x7e007e7e7e007e7eULL,
        0xb400b4b4b400b4b4ULL, 0x2800282828002828ULL, 0x5500555555005555ULL,
        0x6800686868006868ULL, 0x5000505050005050ULL, 0xbe00bebebe00bebeULL,
        0xd000d0d0d000d0d0ULL, 0xc400c4c4c400c4c4ULL, 0x3100313131003131ULL,
        0xcb00cbcbcb00cbcbULL, 0x2a002a2a2a002a2aULL, 0xad00adadad00adadULL,
        0x0f000f0f0f000f0fULL, 0xca00cacaca00cacaULL, 0x7000707070007070ULL,
        0xff00ffffff00ffffULL, 0x3200323232003232ULL, 0x6900696969006969ULL,
        0x0800080808000808ULL, 0x6200626262006262ULL, 0x0000000000000000ULL,
        0x2400242424002424ULL, 0xd100d1d1d100d1d1ULL, 0xfb00fbfbfb00fbfbULL,
        0xba00bababa00babaULL, 0xed00ededed00ededULL, 0x4500454545004545ULL,
        0x8100818181008181ULL, 0x7300737373007373ULL, 0x6d006d6d6d006d6dULL,
        0x8400848484008484ULL, 0x9f009f9f9f009f9fULL, 0xee00eeeeee00eeeeULL,
        0x4a004a4a4a004a4aULL, 0xc300c3c3c300c3c3ULL, 0x2e002e2e2e002e2eULL,
        0xc100c1c1c100c1c1ULL, 0x0100010101000101ULL, 0xe600e6e6e600e6e6ULL,
        0x2500252525002525ULL, 0x4800484848004848ULL, 0x9900999999009999ULL,
        0xb900b9b9b900b9b9ULL, 0xb300b3b3b300b3b3ULL, 0x7b007b7b7b007b7bULL,
        0xf900f9f9f900f9f9ULL, 0xce00cecece00ceceULL, 0xbf00bfbfbf00bfbfULL,
        0xdf00dfdfdf00dfdfULL, 0x7100717171007171ULL, 0x2900292929002929ULL,
        0xcd00cdcdcd00cdcdULL, 0x6c006c6c6c006c6cULL, 0x1300131313001313ULL,
        0x6400646464006464ULL, 0x9b009b9b9b009b9bULL, 0x6300636363006363ULL,
        0x9d009d9d9d009d9dULL, 0xc000c0c0c000c0c0ULL, 0x4b004b4b4b004b4bULL,
        0xb700b7b7b700b7b7ULL, 0xa500a5a5a500a5a5ULL, 0x8900898989008989ULL,
        0x5f005f5f5f005f5fULL, 0xb100b1b1b100b1b1ULL, 0x1700171717001717ULL,
        0xf400f4f4f400f4f4ULL, 0xbc00bcbcbc00bcbcULL, 0xd300d3d3d300d3d3ULL,
        0x4600464646004646ULL, 0xcf00cfcfcf00cfcfULL, 0x3700373737003737ULL,
        0x5e005e5e5e005e5eULL, 0x4700474747004747ULL, 0x9400949494009494ULL,
        0xfa00fafafa00fafaULL, 0xfc00fcfcfc00fcfcULL, 0x5b005b5b5b005b5bULL,
        0x9700979797009797ULL, 0xfe00fefefe00fefeULL, 0x5a005a5a5a005a5aULL,
        0xac00acacac00acacULL, 0x3c003c3c3c003c3cULL, 0x4c004c4c4c004c4cULL,
        0x0300030303000303ULL, 0x3500353535003535ULL, 0xf300f3f3f300f3f3ULL,
        0x2300232323002323ULL, 0xb800b8b8b800b8b8ULL, 0x5d005d5d5d005d5dULL,
        0x6a006a6a6a006a6aULL, 0x9200929292009292ULL, 0xd500d5d5d500d5d5ULL,
        0x2100212121002121ULL, 0x4400444444004444ULL, 0x5100515151005151ULL,
        0xc600c6c6c600c6c6ULL, 0x7d007d7d7d007d7dULL, 0x3900393939003939ULL,
        0x8300838383008383ULL, 0xdc00dcdcdc00dcdcULL, 0xaa00aaaaaa00aaaaULL,
        0x7c007c7c7c007c7cULL, 0x7700777777007777ULL, 0x5600565656005656ULL,
        0x0500050505000505ULL, 0x1b001b1b1b001b1bULL, 0xa400a4a4a400a4a4ULL,
        0x1500151515001515ULL, 0x3400343434003434ULL, 0x1e001e1e1e001e1eULL,
        0x1c001c1c1c001c1cULL, 0xf800f8f8f800f8f8ULL, 0x5200525252005252ULL,
        0x2000202020002020ULL, 0x1400141414001414ULL, 0xe900e9e9e900e9e9ULL,
        0xbd00bdbdbd00bdbdULL, 0xdd00dddddd00ddddULL, 0xe400e4e4e400e4e4ULL,
        0xa100a1a1a100a1a1ULL, 0xe000e0e0e000e0e0ULL, 0x8a008a8a8a008a8aULL,
        0xf100f1f1f100f1f1ULL, 0xd600d6d6d600d6d6ULL, 0x7a007a7a7a007a7aULL,
        0xbb00bbbbbb00bbbbULL, 0xe300e3e3e300e3e3ULL, 0x4000404040004040ULL,
        0x4f004f4f4f004f4fULL,
};

__visible const u64 camellia_sp44044404[256] = {
        0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL,
        0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL,
        0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL,
        0x6b6b006b6b6b006bULL, 0x4545004545450045ULL, 0xa5a500a5a5a500a5ULL,
        0xeded00ededed00edULL, 0x4f4f004f4f4f004fULL, 0x1d1d001d1d1d001dULL,
        0x9292009292920092ULL, 0x8686008686860086ULL, 0xafaf00afafaf00afULL,
        0x7c7c007c7c7c007cULL, 0x1f1f001f1f1f001fULL, 0x3e3e003e3e3e003eULL,
        0xdcdc00dcdcdc00dcULL, 0x5e5e005e5e5e005eULL, 0x0b0b000b0b0b000bULL,
        0xa6a600a6a6a600a6ULL, 0x3939003939390039ULL, 0xd5d500d5d5d500d5ULL,
        0x5d5d005d5d5d005dULL, 0xd9d900d9d9d900d9ULL, 0x5a5a005a5a5a005aULL,
        0x5151005151510051ULL, 0x6c6c006c6c6c006cULL, 0x8b8b008b8b8b008bULL,
        0x9a9a009a9a9a009aULL, 0xfbfb00fbfbfb00fbULL, 0xb0b000b0b0b000b0ULL,
        0x7474007474740074ULL, 0x2b2b002b2b2b002bULL, 0xf0f000f0f0f000f0ULL,
        0x8484008484840084ULL, 0xdfdf00dfdfdf00dfULL, 0xcbcb00cbcbcb00cbULL,
        0x3434003434340034ULL, 0x7676007676760076ULL, 0x6d6d006d6d6d006dULL,
        0xa9a900a9a9a900a9ULL, 0xd1d100d1d1d100d1ULL, 0x0404000404040004ULL,
        0x1414001414140014ULL, 0x3a3a003a3a3a003aULL, 0xdede00dedede00deULL,
        0x1111001111110011ULL, 0x3232003232320032ULL, 0x9c9c009c9c9c009cULL,
        0x5353005353530053ULL, 0xf2f200f2f2f200f2ULL, 0xfefe00fefefe00feULL,
        0xcfcf00cfcfcf00cfULL, 0xc3c300c3c3c300c3ULL, 0x7a7a007a7a7a007aULL,
        0x2424002424240024ULL, 0xe8e800e8e8e800e8ULL, 0x6060006060600060ULL,
        0x6969006969690069ULL, 0xaaaa00aaaaaa00aaULL, 0xa0a000a0a0a000a0ULL,
        0xa1a100a1a1a100a1ULL, 0x6262006262620062ULL, 0x5454005454540054ULL,
        0x1e1e001e1e1e001eULL, 0xe0e000e0e0e000e0ULL, 0x6464006464640064ULL,
        0x1010001010100010ULL, 0x0000000000000000ULL, 0xa3a300a3a3a300a3ULL,
        0x7575007575750075ULL, 0x8a8a008a8a8a008aULL, 0xe6e600e6e6e600e6ULL,
        0x0909000909090009ULL, 0xdddd00dddddd00ddULL, 0x8787008787870087ULL,
        0x8383008383830083ULL, 0xcdcd00cdcdcd00cdULL, 0x9090009090900090ULL,
        0x7373007373730073ULL, 0xf6f600f6f6f600f6ULL, 0x9d9d009d9d9d009dULL,
        0xbfbf00bfbfbf00bfULL, 0x5252005252520052ULL, 0xd8d800d8d8d800d8ULL,
        0xc8c800c8c8c800c8ULL, 0xc6c600c6c6c600c6ULL, 0x8181008181810081ULL,
        0x6f6f006f6f6f006fULL, 0x1313001313130013ULL, 0x6363006363630063ULL,
        0xe9e900e9e9e900e9ULL, 0xa7a700a7a7a700a7ULL, 0x9f9f009f9f9f009fULL,
        0xbcbc00bcbcbc00bcULL, 0x2929002929290029ULL, 0xf9f900f9f9f900f9ULL,
        0x2f2f002f2f2f002fULL, 0xb4b400b4b4b400b4ULL, 0x7878007878780078ULL,
        0x0606000606060006ULL, 0xe7e700e7e7e700e7ULL, 0x7171007171710071ULL,
        0xd4d400d4d4d400d4ULL, 0xabab00ababab00abULL, 0x8888008888880088ULL,
        0x8d8d008d8d8d008dULL, 0x7272007272720072ULL, 0xb9b900b9b9b900b9ULL,
        0xf8f800f8f8f800f8ULL, 0xacac00acacac00acULL, 0x3636003636360036ULL,
        0x2a2a002a2a2a002aULL, 0x3c3c003c3c3c003cULL, 0xf1f100f1f1f100f1ULL,
        0x4040004040400040ULL, 0xd3d300d3d3d300d3ULL, 0xbbbb00bbbbbb00bbULL,
        0x4343004343430043ULL, 0x1515001515150015ULL, 0xadad00adadad00adULL,
        0x7777007777770077ULL, 0x8080008080800080ULL, 0x8282008282820082ULL,
        0xecec00ececec00ecULL, 0x2727002727270027ULL, 0xe5e500e5e5e500e5ULL,
        0x8585008585850085ULL, 0x3535003535350035ULL, 0x0c0c000c0c0c000cULL,
        0x4141004141410041ULL, 0xefef00efefef00efULL, 0x9393009393930093ULL,
        0x1919001919190019ULL, 0x2121002121210021ULL, 0x0e0e000e0e0e000eULL,
        0x4e4e004e4e4e004eULL, 0x6565006565650065ULL, 0xbdbd00bdbdbd00bdULL,
        0xb8b800b8b8b800b8ULL, 0x8f8f008f8f8f008fULL, 0xebeb00ebebeb00ebULL,
        0xcece00cecece00ceULL, 0x3030003030300030ULL, 0x5f5f005f5f5f005fULL,
        0xc5c500c5c5c500c5ULL, 0x1a1a001a1a1a001aULL, 0xe1e100e1e1e100e1ULL,
        0xcaca00cacaca00caULL, 0x4747004747470047ULL, 0x3d3d003d3d3d003dULL,
        0x0101000101010001ULL, 0xd6d600d6d6d600d6ULL, 0x5656005656560056ULL,
        0x4d4d004d4d4d004dULL, 0x0d0d000d0d0d000dULL, 0x6666006666660066ULL,
        0xcccc00cccccc00ccULL, 0x2d2d002d2d2d002dULL, 0x1212001212120012ULL,
        0x2020002020200020ULL, 0xb1b100b1b1b100b1ULL, 0x9999009999990099ULL,
        0x4c4c004c4c4c004cULL, 0xc2c200c2c2c200c2ULL, 0x7e7e007e7e7e007eULL,
        0x0505000505050005ULL, 0xb7b700b7b7b700b7ULL, 0x3131003131310031ULL,
        0x1717001717170017ULL, 0xd7d700d7d7d700d7ULL, 0x5858005858580058ULL,
        0x6161006161610061ULL, 0x1b1b001b1b1b001bULL, 0x1c1c001c1c1c001cULL,
        0x0f0f000f0f0f000fULL, 0x1616001616160016ULL, 0x1818001818180018ULL,
        0x2222002222220022ULL, 0x4444004444440044ULL, 0xb2b200b2b2b200b2ULL,
        0xb5b500b5b5b500b5ULL, 0x9191009191910091ULL, 0x0808000808080008ULL,
        0xa8a800a8a8a800a8ULL, 0xfcfc00fcfcfc00fcULL, 0x5050005050500050ULL,
        0xd0d000d0d0d000d0ULL, 0x7d7d007d7d7d007dULL, 0x8989008989890089ULL,
        0x9797009797970097ULL, 0x5b5b005b5b5b005bULL, 0x9595009595950095ULL,
        0xffff00ffffff00ffULL, 0xd2d200d2d2d200d2ULL, 0xc4c400c4c4c400c4ULL,
        0x4848004848480048ULL, 0xf7f700f7f7f700f7ULL, 0xdbdb00dbdbdb00dbULL,
        0x0303000303030003ULL, 0xdada00dadada00daULL, 0x3f3f003f3f3f003fULL,
        0x9494009494940094ULL, 0x5c5c005c5c5c005cULL, 0x0202000202020002ULL,
        0x4a4a004a4a4a004aULL, 0x3333003333330033ULL, 0x6767006767670067ULL,
        0xf3f300f3f3f300f3ULL, 0x7f7f007f7f7f007fULL, 0xe2e200e2e2e200e2ULL,
        0x9b9b009b9b9b009bULL, 0x2626002626260026ULL, 0x3737003737370037ULL,
        0x3b3b003b3b3b003bULL, 0x9696009696960096ULL, 0x4b4b004b4b4b004bULL,
        0xbebe00bebebe00beULL, 0x2e2e002e2e2e002eULL, 0x7979007979790079ULL,
        0x8c8c008c8c8c008cULL, 0x6e6e006e6e6e006eULL, 0x8e8e008e8e8e008eULL,
        0xf5f500f5f5f500f5ULL, 0xb6b600b6b6b600b6ULL, 0xfdfd00fdfdfd00fdULL,
        0x5959005959590059ULL, 0x9898009898980098ULL, 0x6a6a006a6a6a006aULL,
        0x4646004646460046ULL, 0xbaba00bababa00baULL, 0x2525002525250025ULL,
        0x4242004242420042ULL, 0xa2a200a2a2a200a2ULL, 0xfafa00fafafa00faULL,
        0x0707000707070007ULL, 0x5555005555550055ULL, 0xeeee00eeeeee00eeULL,
        0x0a0a000a0a0a000aULL, 0x4949004949490049ULL, 0x6868006868680068ULL,
        0x3838003838380038ULL, 0xa4a400a4a4a400a4ULL, 0x2828002828280028ULL,
        0x7b7b007b7b7b007bULL, 0xc9c900c9c9c900c9ULL, 0xc1c100c1c1c100c1ULL,
        0xe3e300e3e3e300e3ULL, 0xf4f400f4f4f400f4ULL, 0xc7c700c7c7c700c7ULL,
        0x9e9e009e9e9e009eULL,
};

__visible const u64 camellia_sp11101110[256] = {
        0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL,
        0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL,
        0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL,
        0x8585850085858500ULL, 0x5757570057575700ULL, 0x3535350035353500ULL,
        0xeaeaea00eaeaea00ULL, 0x0c0c0c000c0c0c00ULL, 0xaeaeae00aeaeae00ULL,
        0x4141410041414100ULL, 0x2323230023232300ULL, 0xefefef00efefef00ULL,
        0x6b6b6b006b6b6b00ULL, 0x9393930093939300ULL, 0x4545450045454500ULL,
        0x1919190019191900ULL, 0xa5a5a500a5a5a500ULL, 0x2121210021212100ULL,
        0xededed00ededed00ULL, 0x0e0e0e000e0e0e00ULL, 0x4f4f4f004f4f4f00ULL,
        0x4e4e4e004e4e4e00ULL, 0x1d1d1d001d1d1d00ULL, 0x6565650065656500ULL,
        0x9292920092929200ULL, 0xbdbdbd00bdbdbd00ULL, 0x8686860086868600ULL,
        0xb8b8b800b8b8b800ULL, 0xafafaf00afafaf00ULL, 0x8f8f8f008f8f8f00ULL,
        0x7c7c7c007c7c7c00ULL, 0xebebeb00ebebeb00ULL, 0x1f1f1f001f1f1f00ULL,
        0xcecece00cecece00ULL, 0x3e3e3e003e3e3e00ULL, 0x3030300030303000ULL,
        0xdcdcdc00dcdcdc00ULL, 0x5f5f5f005f5f5f00ULL, 0x5e5e5e005e5e5e00ULL,
        0xc5c5c500c5c5c500ULL, 0x0b0b0b000b0b0b00ULL, 0x1a1a1a001a1a1a00ULL,
        0xa6a6a600a6a6a600ULL, 0xe1e1e100e1e1e100ULL, 0x3939390039393900ULL,
        0xcacaca00cacaca00ULL, 0xd5d5d500d5d5d500ULL, 0x4747470047474700ULL,
        0x5d5d5d005d5d5d00ULL, 0x3d3d3d003d3d3d00ULL, 0xd9d9d900d9d9d900ULL,
        0x0101010001010100ULL, 0x5a5a5a005a5a5a00ULL, 0xd6d6d600d6d6d600ULL,
        0x5151510051515100ULL, 0x5656560056565600ULL, 0x6c6c6c006c6c6c00ULL,
        0x4d4d4d004d4d4d00ULL, 0x8b8b8b008b8b8b00ULL, 0x0d0d0d000d0d0d00ULL,
        0x9a9a9a009a9a9a00ULL, 0x6666660066666600ULL, 0xfbfbfb00fbfbfb00ULL,
        0xcccccc00cccccc00ULL, 0xb0b0b000b0b0b000ULL, 0x2d2d2d002d2d2d00ULL,
        0x7474740074747400ULL, 0x1212120012121200ULL, 0x2b2b2b002b2b2b00ULL,
        0x2020200020202000ULL, 0xf0f0f000f0f0f000ULL, 0xb1b1b100b1b1b100ULL,
        0x8484840084848400ULL, 0x9999990099999900ULL, 0xdfdfdf00dfdfdf00ULL,
        0x4c4c4c004c4c4c00ULL, 0xcbcbcb00cbcbcb00ULL, 0xc2c2c200c2c2c200ULL,
        0x3434340034343400ULL, 0x7e7e7e007e7e7e00ULL, 0x7676760076767600ULL,
        0x0505050005050500ULL, 0x6d6d6d006d6d6d00ULL, 0xb7b7b700b7b7b700ULL,
        0xa9a9a900a9a9a900ULL, 0x3131310031313100ULL, 0xd1d1d100d1d1d100ULL,
        0x1717170017171700ULL, 0x0404040004040400ULL, 0xd7d7d700d7d7d700ULL,
        0x1414140014141400ULL, 0x5858580058585800ULL, 0x3a3a3a003a3a3a00ULL,
        0x6161610061616100ULL, 0xdedede00dedede00ULL, 0x1b1b1b001b1b1b00ULL,
        0x1111110011111100ULL, 0x1c1c1c001c1c1c00ULL, 0x3232320032323200ULL,
        0x0f0f0f000f0f0f00ULL, 0x9c9c9c009c9c9c00ULL, 0x1616160016161600ULL,
        0x5353530053535300ULL, 0x1818180018181800ULL, 0xf2f2f200f2f2f200ULL,
        0x2222220022222200ULL, 0xfefefe00fefefe00ULL, 0x4444440044444400ULL,
        0xcfcfcf00cfcfcf00ULL, 0xb2b2b200b2b2b200ULL, 0xc3c3c300c3c3c300ULL,
        0xb5b5b500b5b5b500ULL, 0x7a7a7a007a7a7a00ULL, 0x9191910091919100ULL,
        0x2424240024242400ULL, 0x0808080008080800ULL, 0xe8e8e800e8e8e800ULL,
        0xa8a8a800a8a8a800ULL, 0x6060600060606000ULL, 0xfcfcfc00fcfcfc00ULL,
        0x6969690069696900ULL, 0x5050500050505000ULL, 0xaaaaaa00aaaaaa00ULL,
        0xd0d0d000d0d0d000ULL, 0xa0a0a000a0a0a000ULL, 0x7d7d7d007d7d7d00ULL,
        0xa1a1a100a1a1a100ULL, 0x8989890089898900ULL, 0x6262620062626200ULL,
        0x9797970097979700ULL, 0x5454540054545400ULL, 0x5b5b5b005b5b5b00ULL,
        0x1e1e1e001e1e1e00ULL, 0x9595950095959500ULL, 0xe0e0e000e0e0e000ULL,
        0xffffff00ffffff00ULL, 0x6464640064646400ULL, 0xd2d2d200d2d2d200ULL,
        0x1010100010101000ULL, 0xc4c4c400c4c4c400ULL, 0x0000000000000000ULL,
        0x4848480048484800ULL, 0xa3a3a300a3a3a300ULL, 0xf7f7f700f7f7f700ULL,
        0x7575750075757500ULL, 0xdbdbdb00dbdbdb00ULL, 0x8a8a8a008a8a8a00ULL,
        0x0303030003030300ULL, 0xe6e6e600e6e6e600ULL, 0xdadada00dadada00ULL,
        0x0909090009090900ULL, 0x3f3f3f003f3f3f00ULL, 0xdddddd00dddddd00ULL,
        0x9494940094949400ULL, 0x8787870087878700ULL, 0x5c5c5c005c5c5c00ULL,
        0x8383830083838300ULL, 0x0202020002020200ULL, 0xcdcdcd00cdcdcd00ULL,
        0x4a4a4a004a4a4a00ULL, 0x9090900090909000ULL, 0x3333330033333300ULL,
        0x7373730073737300ULL, 0x6767670067676700ULL, 0xf6f6f600f6f6f600ULL,
        0xf3f3f300f3f3f300ULL, 0x9d9d9d009d9d9d00ULL, 0x7f7f7f007f7f7f00ULL,
        0xbfbfbf00bfbfbf00ULL, 0xe2e2e200e2e2e200ULL, 0x5252520052525200ULL,
        0x9b9b9b009b9b9b00ULL, 0xd8d8d800d8d8d800ULL, 0x2626260026262600ULL,
        0xc8c8c800c8c8c800ULL, 0x3737370037373700ULL, 0xc6c6c600c6c6c600ULL,
        0x3b3b3b003b3b3b00ULL, 0x8181810081818100ULL, 0x9696960096969600ULL,
        0x6f6f6f006f6f6f00ULL, 0x4b4b4b004b4b4b00ULL, 0x1313130013131300ULL,
        0xbebebe00bebebe00ULL, 0x6363630063636300ULL, 0x2e2e2e002e2e2e00ULL,
        0xe9e9e900e9e9e900ULL, 0x7979790079797900ULL, 0xa7a7a700a7a7a700ULL,
        0x8c8c8c008c8c8c00ULL, 0x9f9f9f009f9f9f00ULL, 0x6e6e6e006e6e6e00ULL,
        0xbcbcbc00bcbcbc00ULL, 0x8e8e8e008e8e8e00ULL, 0x2929290029292900ULL,
        0xf5f5f500f5f5f500ULL, 0xf9f9f900f9f9f900ULL, 0xb6b6b600b6b6b600ULL,
        0x2f2f2f002f2f2f00ULL, 0xfdfdfd00fdfdfd00ULL, 0xb4b4b400b4b4b400ULL,
        0x5959590059595900ULL, 0x7878780078787800ULL, 0x9898980098989800ULL,
        0x0606060006060600ULL, 0x6a6a6a006a6a6a00ULL, 0xe7e7e700e7e7e700ULL,
        0x4646460046464600ULL, 0x7171710071717100ULL, 0xbababa00bababa00ULL,
        0xd4d4d400d4d4d400ULL, 0x2525250025252500ULL, 0xababab00ababab00ULL,
        0x4242420042424200ULL, 0x8888880088888800ULL, 0xa2a2a200a2a2a200ULL,
        0x8d8d8d008d8d8d00ULL, 0xfafafa00fafafa00ULL, 0x7272720072727200ULL,
        0x0707070007070700ULL, 0xb9b9b900b9b9b900ULL, 0x5555550055555500ULL,
        0xf8f8f800f8f8f800ULL, 0xeeeeee00eeeeee00ULL, 0xacacac00acacac00ULL,
        0x0a0a0a000a0a0a00ULL, 0x3636360036363600ULL, 0x4949490049494900ULL,
        0x2a2a2a002a2a2a00ULL, 0x6868680068686800ULL, 0x3c3c3c003c3c3c00ULL,
        0x3838380038383800ULL, 0xf1f1f100f1f1f100ULL, 0xa4a4a400a4a4a400ULL,
        0x4040400040404000ULL, 0x2828280028282800ULL, 0xd3d3d300d3d3d300ULL,
        0x7b7b7b007b7b7b00ULL, 0xbbbbbb00bbbbbb00ULL, 0xc9c9c900c9c9c900ULL,
        0x4343430043434300ULL, 0xc1c1c100c1c1c100ULL, 0x1515150015151500ULL,
        0xe3e3e300e3e3e300ULL, 0xadadad00adadad00ULL, 0xf4f4f400f4f4f400ULL,
        0x7777770077777700ULL, 0xc7c7c700c7c7c700ULL, 0x8080800080808000ULL,
        0x9e9e9e009e9e9e00ULL,
};

/* key constants */
#define CAMELLIA_SIGMA1L (0xA09E667FL)
#define CAMELLIA_SIGMA1R (0x3BCC908BL)
#define CAMELLIA_SIGMA2L (0xB67AE858L)
#define CAMELLIA_SIGMA2R (0x4CAA73B2L)
#define CAMELLIA_SIGMA3L (0xC6EF372FL)
#define CAMELLIA_SIGMA3R (0xE94F82BEL)
#define CAMELLIA_SIGMA4L (0x54FF53A5L)
#define CAMELLIA_SIGMA4R (0xF1D36F1CL)
#define CAMELLIA_SIGMA5L (0x10E527FAL)
#define CAMELLIA_SIGMA5R (0xDE682D1DL)
#define CAMELLIA_SIGMA6L (0xB05688C2L)
#define CAMELLIA_SIGMA6R (0xB3E6C1FDL)

/* macros */
#define ROLDQ(l, r, bits) ({ \
        u64 t = l;                                        \
        l = (l << bits) | (r >> (64 - bits));                \
        r = (r << bits) | (t >> (64 - bits));                \
})

#define CAMELLIA_F(x, kl, kr, y) ({ \
        u64 ii = x ^ (((u64)kl << 32) | kr);                                \
        y = camellia_sp11101110[(uint8_t)ii];                                \
        y ^= camellia_sp44044404[(uint8_t)(ii >> 8)];                        \
        ii >>= 16;                                                        \
        y ^= camellia_sp30333033[(uint8_t)ii];                                \
        y ^= camellia_sp02220222[(uint8_t)(ii >> 8)];                        \
        ii >>= 16;                                                        \
        y ^= camellia_sp00444404[(uint8_t)ii];                                \
        y ^= camellia_sp03303033[(uint8_t)(ii >> 8)];                        \
        ii >>= 16;                                                        \
        y ^= camellia_sp22000222[(uint8_t)ii];                                \
        y ^= camellia_sp10011110[(uint8_t)(ii >> 8)];                        \
        y = ror64(y, 32);                                                \
})

#define SET_SUBKEY_LR(INDEX, sRL) (subkey[(INDEX)] = ror64((sRL), 32))

static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
{
        u64 kw4, tt;
        u32 dw, tl, tr;

        /* absorb kw2 to other subkeys */
        /* round 2 */
        subRL[3] ^= subRL[1];
        /* round 4 */
        subRL[5] ^= subRL[1];
        /* round 6 */
        subRL[7] ^= subRL[1];

        subRL[1] ^= (subRL[1] & ~subRL[9]) << 32;
        /* modified for FLinv(kl2) */
        dw = (subRL[1] & subRL[9]) >> 32;
        subRL[1] ^= rol32(dw, 1);

        /* round 8 */
        subRL[11] ^= subRL[1];
        /* round 10 */
        subRL[13] ^= subRL[1];
        /* round 12 */
        subRL[15] ^= subRL[1];

        subRL[1] ^= (subRL[1] & ~subRL[17]) << 32;
        /* modified for FLinv(kl4) */
        dw = (subRL[1] & subRL[17]) >> 32;
        subRL[1] ^= rol32(dw, 1);

        /* round 14 */
        subRL[19] ^= subRL[1];
        /* round 16 */
        subRL[21] ^= subRL[1];
        /* round 18 */
        subRL[23] ^= subRL[1];

        if (max == 24) {
                /* kw3 */
                subRL[24] ^= subRL[1];

                /* absorb kw4 to other subkeys */
                kw4 = subRL[25];
        } else {
                subRL[1] ^= (subRL[1] & ~subRL[25]) << 32;
                /* modified for FLinv(kl6) */
                dw = (subRL[1] & subRL[25]) >> 32;
                subRL[1] ^= rol32(dw, 1);

                /* round 20 */
                subRL[27] ^= subRL[1];
                /* round 22 */
                subRL[29] ^= subRL[1];
                /* round 24 */
                subRL[31] ^= subRL[1];
                /* kw3 */
                subRL[32] ^= subRL[1];

                /* absorb kw4 to other subkeys */
                kw4 = subRL[33];
                /* round 23 */
                subRL[30] ^= kw4;
                /* round 21 */
                subRL[28] ^= kw4;
                /* round 19 */
                subRL[26] ^= kw4;

                kw4 ^= (kw4 & ~subRL[24]) << 32;
                /* modified for FL(kl5) */
                dw = (kw4 & subRL[24]) >> 32;
                kw4 ^= rol32(dw, 1);
        }

        /* round 17 */
        subRL[22] ^= kw4;
        /* round 15 */
        subRL[20] ^= kw4;
        /* round 13 */
        subRL[18] ^= kw4;

        kw4 ^= (kw4 & ~subRL[16]) << 32;
        /* modified for FL(kl3) */
        dw = (kw4 & subRL[16]) >> 32;
        kw4 ^= rol32(dw, 1);

        /* round 11 */
        subRL[14] ^= kw4;
        /* round 9 */
        subRL[12] ^= kw4;
        /* round 7 */
        subRL[10] ^= kw4;

        kw4 ^= (kw4 & ~subRL[8]) << 32;
        /* modified for FL(kl1) */
        dw = (kw4 & subRL[8]) >> 32;
        kw4 ^= rol32(dw, 1);

        /* round 5 */
        subRL[6] ^= kw4;
        /* round 3 */
        subRL[4] ^= kw4;
        /* round 1 */
        subRL[2] ^= kw4;
        /* kw1 */
        subRL[0] ^= kw4;

        /* key XOR is end of F-function */
        SET_SUBKEY_LR(0, subRL[0] ^ subRL[2]);                        /* kw1 */
        SET_SUBKEY_LR(2, subRL[3]);                                /* round 1 */
        SET_SUBKEY_LR(3, subRL[2] ^ subRL[4]);                        /* round 2 */
        SET_SUBKEY_LR(4, subRL[3] ^ subRL[5]);                        /* round 3 */
        SET_SUBKEY_LR(5, subRL[4] ^ subRL[6]);                        /* round 4 */
        SET_SUBKEY_LR(6, subRL[5] ^ subRL[7]);                        /* round 5 */

        tl = (subRL[10] >> 32) ^ (subRL[10] & ~subRL[8]);
        dw = tl & (subRL[8] >> 32);                                /* FL(kl1) */
        tr = subRL[10] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(7, subRL[6] ^ tt);                        /* round 6 */
        SET_SUBKEY_LR(8, subRL[8]);                                /* FL(kl1) */
        SET_SUBKEY_LR(9, subRL[9]);                                /* FLinv(kl2) */

        tl = (subRL[7] >> 32) ^ (subRL[7] & ~subRL[9]);
        dw = tl & (subRL[9] >> 32);                                /* FLinv(kl2) */
        tr = subRL[7] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(10, subRL[11] ^ tt);                        /* round 7 */
        SET_SUBKEY_LR(11, subRL[10] ^ subRL[12]);                /* round 8 */
        SET_SUBKEY_LR(12, subRL[11] ^ subRL[13]);                /* round 9 */
        SET_SUBKEY_LR(13, subRL[12] ^ subRL[14]);                /* round 10 */
        SET_SUBKEY_LR(14, subRL[13] ^ subRL[15]);                /* round 11 */

        tl = (subRL[18] >> 32) ^ (subRL[18] & ~subRL[16]);
        dw = tl & (subRL[16] >> 32);                                /* FL(kl3) */
        tr = subRL[18] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(15, subRL[14] ^ tt);                        /* round 12 */
        SET_SUBKEY_LR(16, subRL[16]);                                /* FL(kl3) */
        SET_SUBKEY_LR(17, subRL[17]);                                /* FLinv(kl4) */

        tl = (subRL[15] >> 32) ^ (subRL[15] & ~subRL[17]);
        dw = tl & (subRL[17] >> 32);                                /* FLinv(kl4) */
        tr = subRL[15] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(18, subRL[19] ^ tt);                        /* round 13 */
        SET_SUBKEY_LR(19, subRL[18] ^ subRL[20]);                /* round 14 */
        SET_SUBKEY_LR(20, subRL[19] ^ subRL[21]);                /* round 15 */
        SET_SUBKEY_LR(21, subRL[20] ^ subRL[22]);                /* round 16 */
        SET_SUBKEY_LR(22, subRL[21] ^ subRL[23]);                /* round 17 */

        if (max == 24) {
                SET_SUBKEY_LR(23, subRL[22]);                        /* round 18 */
                SET_SUBKEY_LR(24, subRL[24] ^ subRL[23]);        /* kw3 */
        } else {
                tl = (subRL[26] >> 32) ^ (subRL[26] & ~subRL[24]);
                dw = tl & (subRL[24] >> 32);                        /* FL(kl5) */
                tr = subRL[26] ^ rol32(dw, 1);
                tt = (tr | ((u64)tl << 32));

                SET_SUBKEY_LR(23, subRL[22] ^ tt);                /* round 18 */
                SET_SUBKEY_LR(24, subRL[24]);                        /* FL(kl5) */
                SET_SUBKEY_LR(25, subRL[25]);                        /* FLinv(kl6) */

                tl = (subRL[23] >> 32) ^ (subRL[23] & ~subRL[25]);
                dw = tl & (subRL[25] >> 32);                        /* FLinv(kl6) */
                tr = subRL[23] ^ rol32(dw, 1);
                tt = (tr | ((u64)tl << 32));

                SET_SUBKEY_LR(26, subRL[27] ^ tt);                /* round 19 */
                SET_SUBKEY_LR(27, subRL[26] ^ subRL[28]);        /* round 20 */
                SET_SUBKEY_LR(28, subRL[27] ^ subRL[29]);        /* round 21 */
                SET_SUBKEY_LR(29, subRL[28] ^ subRL[30]);        /* round 22 */
                SET_SUBKEY_LR(30, subRL[29] ^ subRL[31]);        /* round 23 */
                SET_SUBKEY_LR(31, subRL[30]);                        /* round 24 */
                SET_SUBKEY_LR(32, subRL[32] ^ subRL[31]);        /* kw3 */
        }
}

static void camellia_setup128(const unsigned char *key, u64 *subkey)
{
        u64 kl, kr, ww;
        u64 subRL[26];

        /**
         *  k == kl || kr (|| is concatenation)
         */
        kl = get_unaligned_be64(key);
        kr = get_unaligned_be64(key + 8);

        /* generate KL dependent subkeys */
        /* kw1 */
        subRL[0] = kl;
        /* kw2 */
        subRL[1] = kr;

        /* rotation left shift 15bit */
        ROLDQ(kl, kr, 15);

        /* k3 */
        subRL[4] = kl;
        /* k4 */
        subRL[5] = kr;

        /* rotation left shift 15+30bit */
        ROLDQ(kl, kr, 30);

        /* k7 */
        subRL[10] = kl;
        /* k8 */
        subRL[11] = kr;

        /* rotation left shift 15+30+15bit */
        ROLDQ(kl, kr, 15);

        /* k10 */
        subRL[13] = kr;
        /* rotation left shift 15+30+15+17 bit */
        ROLDQ(kl, kr, 17);

        /* kl3 */
        subRL[16] = kl;
        /* kl4 */
        subRL[17] = kr;

        /* rotation left shift 15+30+15+17+17 bit */
        ROLDQ(kl, kr, 17);

        /* k13 */
        subRL[18] = kl;
        /* k14 */
        subRL[19] = kr;

        /* rotation left shift 15+30+15+17+17+17 bit */
        ROLDQ(kl, kr, 17);

        /* k17 */
        subRL[22] = kl;
        /* k18 */
        subRL[23] = kr;

        /* generate KA */
        kl = subRL[0];
        kr = subRL[1];
        CAMELLIA_F(kl, CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R, ww);
        kr ^= ww;
        CAMELLIA_F(kr, CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R, kl);

        /* current status == (kll, klr, w0, w1) */
        CAMELLIA_F(kl, CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R, kr);
        kr ^= ww;
        CAMELLIA_F(kr, CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R, ww);
        kl ^= ww;

        /* generate KA dependent subkeys */
        /* k1, k2 */
        subRL[2] = kl;
        subRL[3] = kr;
        ROLDQ(kl, kr, 15);
        /* k5,k6 */
        subRL[6] = kl;
        subRL[7] = kr;
        ROLDQ(kl, kr, 15);
        /* kl1, kl2 */
        subRL[8] = kl;
        subRL[9] = kr;
        ROLDQ(kl, kr, 15);
        /* k9 */
        subRL[12] = kl;
        ROLDQ(kl, kr, 15);
        /* k11, k12 */
        subRL[14] = kl;
        subRL[15] = kr;
        ROLDQ(kl, kr, 34);
        /* k15, k16 */
        subRL[20] = kl;
        subRL[21] = kr;
        ROLDQ(kl, kr, 17);
        /* kw3, kw4 */
        subRL[24] = kl;
        subRL[25] = kr;

        camellia_setup_tail(subkey, subRL, 24);
}

static void camellia_setup256(const unsigned char *key, u64 *subkey)
{
        u64 kl, kr;                        /* left half of key */
        u64 krl, krr;                        /* right half of key */
        u64 ww;                                /* temporary variables */
        u64 subRL[34];

        /**
         *  key = (kl || kr || krl || krr) (|| is concatenation)
         */
        kl = get_unaligned_be64(key);
        kr = get_unaligned_be64(key + 8);
        krl = get_unaligned_be64(key + 16);
        krr = get_unaligned_be64(key + 24);

        /* generate KL dependent subkeys */
        /* kw1 */
        subRL[0] = kl;
        /* kw2 */
        subRL[1] = kr;
        ROLDQ(kl, kr, 45);
        /* k9 */
        subRL[12] = kl;
        /* k10 */
        subRL[13] = kr;
        ROLDQ(kl, kr, 15);
        /* kl3 */
        subRL[16] = kl;
        /* kl4 */
        subRL[17] = kr;
        ROLDQ(kl, kr, 17);
        /* k17 */
        subRL[22] = kl;
        /* k18 */
        subRL[23] = kr;
        ROLDQ(kl, kr, 34);
        /* k23 */
        subRL[30] = kl;
        /* k24 */
        subRL[31] = kr;

        /* generate KR dependent subkeys */
        ROLDQ(krl, krr, 15);
        /* k3 */
        subRL[4] = krl;
        /* k4 */
        subRL[5] = krr;
        ROLDQ(krl, krr, 15);
        /* kl1 */
        subRL[8] = krl;
        /* kl2 */
        subRL[9] = krr;
        ROLDQ(krl, krr, 30);
        /* k13 */
        subRL[18] = krl;
        /* k14 */
        subRL[19] = krr;
        ROLDQ(krl, krr, 34);
        /* k19 */
        subRL[26] = krl;
        /* k20 */
        subRL[27] = krr;
        ROLDQ(krl, krr, 34);

        /* generate KA */
        kl = subRL[0] ^ krl;
        kr = subRL[1] ^ krr;

        CAMELLIA_F(kl, CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R, ww);
        kr ^= ww;
        CAMELLIA_F(kr, CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R, kl);
        kl ^= krl;
        CAMELLIA_F(kl, CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R, kr);
        kr ^= ww ^ krr;
        CAMELLIA_F(kr, CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R, ww);
        kl ^= ww;

        /* generate KB */
        krl ^= kl;
        krr ^= kr;
        CAMELLIA_F(krl, CAMELLIA_SIGMA5L, CAMELLIA_SIGMA5R, ww);
        krr ^= ww;
        CAMELLIA_F(krr, CAMELLIA_SIGMA6L, CAMELLIA_SIGMA6R, ww);
        krl ^= ww;

        /* generate KA dependent subkeys */
        ROLDQ(kl, kr, 15);
        /* k5 */
        subRL[6] = kl;
        /* k6 */
        subRL[7] = kr;
        ROLDQ(kl, kr, 30);
        /* k11 */
        subRL[14] = kl;
        /* k12 */
        subRL[15] = kr;
        /* rotation left shift 32bit */
        ROLDQ(kl, kr, 32);
        /* kl5 */
        subRL[24] = kl;
        /* kl6 */
        subRL[25] = kr;
        /* rotation left shift 17 from k11,k12 -> k21,k22 */
        ROLDQ(kl, kr, 17);
        /* k21 */
        subRL[28] = kl;
        /* k22 */
        subRL[29] = kr;

        /* generate KB dependent subkeys */
        /* k1 */
        subRL[2] = krl;
        /* k2 */
        subRL[3] = krr;
        ROLDQ(krl, krr, 30);
        /* k7 */
        subRL[10] = krl;
        /* k8 */
        subRL[11] = krr;
        ROLDQ(krl, krr, 30);
        /* k15 */
        subRL[20] = krl;
        /* k16 */
        subRL[21] = krr;
        ROLDQ(krl, krr, 51);
        /* kw3 */
        subRL[32] = krl;
        /* kw4 */
        subRL[33] = krr;

        camellia_setup_tail(subkey, subRL, 32);
}

static void camellia_setup192(const unsigned char *key, u64 *subkey)
{
        unsigned char kk[32];
        u64 krl, krr;

        memcpy(kk, key, 24);
        memcpy((unsigned char *)&krl, key+16, 8);
        krr = ~krl;
        memcpy(kk+24, (unsigned char *)&krr, 8);
        camellia_setup256(kk, subkey);
}

int __camellia_setkey(struct camellia_ctx *cctx, const unsigned char *key,
                      unsigned int key_len)
{
        if (key_len != 16 && key_len != 24 && key_len != 32)
                return -EINVAL;

        cctx->key_length = key_len;

        switch (key_len) {
        case 16:
                camellia_setup128(key, cctx->key_table);
                break;
        case 24:
                camellia_setup192(key, cctx->key_table);
                break;
        case 32:
                camellia_setup256(key, cctx->key_table);
                break;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(__camellia_setkey);

static int camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
                           unsigned int key_len)
{
        return __camellia_setkey(crypto_tfm_ctx(tfm), key, key_len);
}

static int camellia_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key,
                                    unsigned int key_len)
{
        return camellia_setkey(&tfm->base, key, key_len);
}

void camellia_decrypt_cbc_2way(const void *ctx, u8 *dst, const u8 *src)
{
        u8 buf[CAMELLIA_BLOCK_SIZE];
        const u8 *iv = src;

        if (dst == src)
                iv = memcpy(buf, iv, sizeof(buf));
        camellia_dec_blk_2way(ctx, dst, src);
        crypto_xor(dst + CAMELLIA_BLOCK_SIZE, iv, CAMELLIA_BLOCK_SIZE);
}
EXPORT_SYMBOL_GPL(camellia_decrypt_cbc_2way);

static int ecb_encrypt(struct skcipher_request *req)
{
        ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1);
        ECB_BLOCK(2, camellia_enc_blk_2way);
        ECB_BLOCK(1, camellia_enc_blk);
        ECB_WALK_END();
}

static int ecb_decrypt(struct skcipher_request *req)
{
        ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1);
        ECB_BLOCK(2, camellia_dec_blk_2way);
        ECB_BLOCK(1, camellia_dec_blk);
        ECB_WALK_END();
}

static int cbc_encrypt(struct skcipher_request *req)
{
        CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1);
        CBC_ENC_BLOCK(camellia_enc_blk);
        CBC_WALK_END();
}

static int cbc_decrypt(struct skcipher_request *req)
{
        CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1);
        CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way);
        CBC_DEC_BLOCK(1, camellia_dec_blk);
        CBC_WALK_END();
}

static struct crypto_alg camellia_cipher_alg = {
        .cra_name                = "camellia",
        .cra_driver_name        = "camellia-asm",
        .cra_priority                = 200,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = CAMELLIA_BLOCK_SIZE,
        .cra_ctxsize                = sizeof(struct camellia_ctx),
        .cra_module                = THIS_MODULE,
        .cra_u                        = {
                .cipher = {
                        .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE,
                        .cia_max_keysize = CAMELLIA_MAX_KEY_SIZE,
                        .cia_setkey         = camellia_setkey,
                        .cia_encrypt         = camellia_encrypt,
                        .cia_decrypt         = camellia_decrypt
                }
        }
};

static struct skcipher_alg camellia_skcipher_algs[] = {
        {
                .base.cra_name                = "ecb(camellia)",
                .base.cra_driver_name        = "ecb-camellia-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .setkey                        = camellia_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(camellia)",
                .base.cra_driver_name        = "cbc-camellia-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, camellia-asm is slower than original assembler
                 * implementation because excessive uses of 64bit rotate and
                 * left-shifts (which are really slow on P4) needed to store and
                 * handle 128bit block in two 64bit registers.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init camellia_init(void)
{
        int err;

        if (!force && is_blacklisted_cpu()) {
                printk(KERN_INFO
                        "camellia-x86_64: performance on this CPU "
                        "would be suboptimal: disabling "
                        "camellia-x86_64.\n");
                return -ENODEV;
        }

        err = crypto_register_alg(&camellia_cipher_alg);
        if (err)
                return err;

        err = crypto_register_skciphers(camellia_skcipher_algs,
                                        ARRAY_SIZE(camellia_skcipher_algs));
        if (err)
                crypto_unregister_alg(&camellia_cipher_alg);

        return err;
}

static void __exit camellia_fini(void)
{
        crypto_unregister_alg(&camellia_cipher_alg);
        crypto_unregister_skciphers(camellia_skcipher_algs,
                                    ARRAY_SIZE(camellia_skcipher_algs));
}

module_init(camellia_init);
module_exit(camellia_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Camellia Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("camellia");
MODULE_ALIAS_CRYPTO("camellia-asm");






















































































 5081 
















































































   95 














































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
/*
 * include/linux/topology.h
 *
 * Written by: Matthew Dobson, IBM Corporation
 *
 * Copyright (C) 2002, IBM Corp.
 *
 * All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 * NON INFRINGEMENT.  See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 * Send feedback to <colpatch@us.ibm.com>
 */
#ifndef _LINUX_TOPOLOGY_H
#define _LINUX_TOPOLOGY_H

#include <linux/arch_topology.h>
#include <linux/cpumask.h>
#include <linux/nodemask.h>
#include <linux/bitops.h>
#include <linux/mmzone.h>
#include <linux/smp.h>
#include <linux/percpu.h>
#include <asm/topology.h>

#ifndef nr_cpus_node
#define nr_cpus_node(node) cpumask_weight(cpumask_of_node(node))
#endif

int arch_update_cpu_topology(void);

/* Conform to ACPI 2.0 SLIT distance definitions */
#define LOCAL_DISTANCE                10
#define REMOTE_DISTANCE                20
#define DISTANCE_BITS           8
#ifndef node_distance
#define node_distance(from,to)        ((from) == (to) ? LOCAL_DISTANCE : REMOTE_DISTANCE)
#endif
#ifndef RECLAIM_DISTANCE
/*
 * If the distance between nodes in a system is larger than RECLAIM_DISTANCE
 * (in whatever arch specific measurement units returned by node_distance())
 * and node_reclaim_mode is enabled then the VM will only call node_reclaim()
 * on nodes within this distance.
 */
#define RECLAIM_DISTANCE 30
#endif

/*
 * The following tunable allows platforms to override the default node
 * reclaim distance (RECLAIM_DISTANCE) if remote memory accesses are
 * sufficiently fast that the default value actually hurts
 * performance.
 *
 * AMD EPYC machines use this because even though the 2-hop distance
 * is 32 (3.2x slower than a local memory access) performance actually
 * *improves* if allowed to reclaim memory and load balance tasks
 * between NUMA nodes 2-hops apart.
 */
extern int __read_mostly node_reclaim_distance;

#ifndef PENALTY_FOR_NODE_WITH_CPUS
#define PENALTY_FOR_NODE_WITH_CPUS        (1)
#endif

#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DECLARE_PER_CPU(int, numa_node);

#ifndef numa_node_id
/* Returns the number of the current Node. */
static inline int numa_node_id(void)
{
        return raw_cpu_read(numa_node);
}
#endif

#ifndef cpu_to_node
static inline int cpu_to_node(int cpu)
{
        return per_cpu(numa_node, cpu);
}
#endif

#ifndef set_numa_node
static inline void set_numa_node(int node)
{
        this_cpu_write(numa_node, node);
}
#endif

#ifndef set_cpu_numa_node
static inline void set_cpu_numa_node(int cpu, int node)
{
        per_cpu(numa_node, cpu) = node;
}
#endif

#else        /* !CONFIG_USE_PERCPU_NUMA_NODE_ID */

/* Returns the number of the current Node. */
#ifndef numa_node_id
static inline int numa_node_id(void)
{
        return cpu_to_node(raw_smp_processor_id());
}
#endif

#endif        /* [!]CONFIG_USE_PERCPU_NUMA_NODE_ID */

#ifdef CONFIG_HAVE_MEMORYLESS_NODES

/*
 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem().
 */
DECLARE_PER_CPU(int, _numa_mem_);

#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
        this_cpu_write(_numa_mem_, node);
}
#endif

#ifndef numa_mem_id
/* Returns the number of the nearest Node with memory */
static inline int numa_mem_id(void)
{
        return raw_cpu_read(_numa_mem_);
}
#endif

#ifndef cpu_to_mem
static inline int cpu_to_mem(int cpu)
{
        return per_cpu(_numa_mem_, cpu);
}
#endif

#ifndef set_cpu_numa_mem
static inline void set_cpu_numa_mem(int cpu, int node)
{
        per_cpu(_numa_mem_, cpu) = node;
}
#endif

#else        /* !CONFIG_HAVE_MEMORYLESS_NODES */

#ifndef numa_mem_id
/* Returns the number of the nearest Node with memory */
static inline int numa_mem_id(void)
{
        return numa_node_id();
}
#endif

#ifndef cpu_to_mem
static inline int cpu_to_mem(int cpu)
{
        return cpu_to_node(cpu);
}
#endif

#endif        /* [!]CONFIG_HAVE_MEMORYLESS_NODES */

#if defined(topology_die_id) && defined(topology_die_cpumask)
#define TOPOLOGY_DIE_SYSFS
#endif
#if defined(topology_cluster_id) && defined(topology_cluster_cpumask)
#define TOPOLOGY_CLUSTER_SYSFS
#endif
#if defined(topology_book_id) && defined(topology_book_cpumask)
#define TOPOLOGY_BOOK_SYSFS
#endif
#if defined(topology_drawer_id) && defined(topology_drawer_cpumask)
#define TOPOLOGY_DRAWER_SYSFS
#endif

#ifndef topology_physical_package_id
#define topology_physical_package_id(cpu)        ((void)(cpu), -1)
#endif
#ifndef topology_die_id
#define topology_die_id(cpu)                        ((void)(cpu), -1)
#endif
#ifndef topology_cluster_id
#define topology_cluster_id(cpu)                ((void)(cpu), -1)
#endif
#ifndef topology_core_id
#define topology_core_id(cpu)                        ((void)(cpu), 0)
#endif
#ifndef topology_book_id
#define topology_book_id(cpu)                        ((void)(cpu), -1)
#endif
#ifndef topology_drawer_id
#define topology_drawer_id(cpu)                        ((void)(cpu), -1)
#endif
#ifndef topology_ppin
#define topology_ppin(cpu)                        ((void)(cpu), 0ull)
#endif
#ifndef topology_sibling_cpumask
#define topology_sibling_cpumask(cpu)                cpumask_of(cpu)
#endif
#ifndef topology_core_cpumask
#define topology_core_cpumask(cpu)                cpumask_of(cpu)
#endif
#ifndef topology_cluster_cpumask
#define topology_cluster_cpumask(cpu)                cpumask_of(cpu)
#endif
#ifndef topology_die_cpumask
#define topology_die_cpumask(cpu)                cpumask_of(cpu)
#endif
#ifndef topology_book_cpumask
#define topology_book_cpumask(cpu)                cpumask_of(cpu)
#endif
#ifndef topology_drawer_cpumask
#define topology_drawer_cpumask(cpu)                cpumask_of(cpu)
#endif

#if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask)
static inline const struct cpumask *cpu_smt_mask(int cpu)
{
        return topology_sibling_cpumask(cpu);
}
#endif

#ifndef topology_is_primary_thread

static inline bool topology_is_primary_thread(unsigned int cpu)
{
        /*
         * When disabling SMT, the primary thread of the SMT will remain
         * enabled/active. Architectures that have a special primary thread
         * (e.g. x86) need to override this function. Otherwise the first
         * thread in the SMT can be made the primary thread.
         *
         * The sibling cpumask of an offline CPU always contains the CPU
         * itself on architectures using the implementation of
         * CONFIG_GENERIC_ARCH_TOPOLOGY for building their topology.
         * Other architectures not using CONFIG_GENERIC_ARCH_TOPOLOGY for
         * building their topology have to check whether to use this default
         * implementation or to override it.
         */
        return cpu == cpumask_first(topology_sibling_cpumask(cpu));
}
#define topology_is_primary_thread topology_is_primary_thread

#endif

static inline const struct cpumask *cpu_node_mask(int cpu)
{
        return cpumask_of_node(cpu_to_node(cpu));
}

#ifdef CONFIG_NUMA
int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node);
extern const struct cpumask *sched_numa_hop_mask(unsigned int node, unsigned int hops);
#else
static __always_inline int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node)
{
        return cpumask_nth_and(cpu, cpus, cpu_online_mask);
}

static inline const struct cpumask *
sched_numa_hop_mask(unsigned int node, unsigned int hops)
{
        return ERR_PTR(-EOPNOTSUPP);
}
#endif        /* CONFIG_NUMA */

/**
 * for_each_node_numadist() - iterate over nodes in increasing distance
 *                              order, starting from a given node
 * @node: the iteration variable and the starting node.
 * @unvisited: a nodemask to keep track of the unvisited nodes.
 *
 * This macro iterates over NUMA node IDs in increasing distance from the
 * starting @node and yields MAX_NUMNODES when all the nodes have been
 * visited.
 *
 * Note that by the time the loop completes, the @unvisited nodemask will
 * be fully cleared, unless the loop exits early.
 *
 * The difference between for_each_node() and for_each_node_numadist() is
 * that the former allows to iterate over nodes in numerical order, whereas
 * the latter iterates over nodes in increasing order of distance.
 *
 * This complexity of this iterator is O(N^2), where N represents the
 * number of nodes, as each iteration involves scanning all nodes to
 * find the one with the shortest distance.
 *
 * Requires rcu_lock to be held.
 */
#define for_each_node_numadist(node, unvisited)                                        \
        for (int __start = (node),                                                \
             (node) = nearest_node_nodemask((__start), &(unvisited));                \
             (node) < MAX_NUMNODES;                                                \
             node_clear((node), (unvisited)),                                        \
             (node) = nearest_node_nodemask((__start), &(unvisited)))

/**
 * for_each_numa_hop_mask - iterate over cpumasks of increasing NUMA distance
 *                          from a given node.
 * @mask: the iteration variable.
 * @node: the NUMA node to start the search from.
 *
 * Requires rcu_lock to be held.
 *
 * Yields cpu_online_mask for @node == NUMA_NO_NODE.
 */
#define for_each_numa_hop_mask(mask, node)                                       \
        for (unsigned int __hops = 0;                                               \
             mask = (node != NUMA_NO_NODE || __hops) ?                               \
                     sched_numa_hop_mask(node, __hops) :                       \
                     cpu_online_mask,                                               \
             !IS_ERR_OR_NULL(mask);                                               \
             __hops++)

DECLARE_PER_CPU(unsigned long, cpu_scale);

static inline unsigned long topology_get_cpu_scale(int cpu)
{
        return per_cpu(cpu_scale, cpu);
}

void topology_set_cpu_scale(unsigned int cpu, unsigned long capacity);

#endif /* _LINUX_TOPOLOGY_H */
























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 













    2 





























































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
/*
 * Copyright (c) 2006 Damien Bergamini <damien.bergamini@free.fr>
 * Copyright (c) 2006 Sam Leffler, Errno Consulting
 * Copyright (c) 2007 Christoph Hellwig <hch@lst.de>
 * Copyright (c) 2008-2009 Weongyo Jeong <weongyo@freebsd.org>
 * Copyright (c) 2012 Pontus Fuchs <pontus.fuchs@gmail.com>
 *
 * Permission to use, copy, modify, and/or distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

/*
 * This driver is based on the uath driver written by Damien Bergamini for
 * OpenBSD, who did black-box analysis of the Windows binary driver to find
 * out how the hardware works.  It contains a lot magic numbers because of
 * that and only has minimal functionality.
 */
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/completion.h>
#include <linux/firmware.h>
#include <linux/skbuff.h>
#include <linux/usb.h>
#include <net/mac80211.h>

#include "ar5523.h"
#include "ar5523_hw.h"

/*
 * Various supported device vendors/products.
 * UB51: AR5005UG 802.11b/g, UB52: AR5005UX 802.11a/b/g
 */

static int ar5523_submit_rx_cmd(struct ar5523 *ar);
static void ar5523_data_tx_pkt_put(struct ar5523 *ar);

static void ar5523_read_reply(struct ar5523 *ar, struct ar5523_cmd_hdr *hdr,
                              struct ar5523_tx_cmd *cmd)
{
        int dlen, olen;
        __be32 *rp;

        dlen = be32_to_cpu(hdr->len) - sizeof(*hdr);

        if (dlen < 0) {
                WARN_ON(1);
                goto out;
        }

        ar5523_dbg(ar, "Code = %d len = %d\n", be32_to_cpu(hdr->code) & 0xff,
                   dlen);

        rp = (__be32 *)(hdr + 1);
        if (dlen >= sizeof(u32)) {
                olen = be32_to_cpu(rp[0]);
                dlen -= sizeof(u32);
                if (olen == 0) {
                        /* convention is 0 =>'s one word */
                        olen = sizeof(u32);
                }
        } else
                olen = 0;

        if (cmd->odata) {
                if (cmd->olen < olen) {
                        ar5523_err(ar, "olen too small %d < %d\n",
                                   cmd->olen, olen);
                        cmd->olen = 0;
                        cmd->res = -EOVERFLOW;
                } else {
                        cmd->olen = olen;
                        memcpy(cmd->odata, &rp[1], olen);
                        cmd->res = 0;
                }
        }

out:
        complete(&cmd->done);
}

static void ar5523_cmd_rx_cb(struct urb *urb)
{
        struct ar5523 *ar = urb->context;
        struct ar5523_tx_cmd *cmd = &ar->tx_cmd;
        struct ar5523_cmd_hdr *hdr = ar->rx_cmd_buf;
        int dlen;
        u32 code, hdrlen;

        if (urb->status) {
                if (urb->status != -ESHUTDOWN)
                        ar5523_err(ar, "RX USB error %d.\n", urb->status);
                goto skip;
        }

        if (urb->actual_length < sizeof(struct ar5523_cmd_hdr)) {
                ar5523_err(ar, "RX USB too short.\n");
                goto skip;
        }

        ar5523_dbg(ar, "%s code %02x priv %d\n", __func__,
                   be32_to_cpu(hdr->code) & 0xff, hdr->priv);

        code = be32_to_cpu(hdr->code);
        hdrlen = be32_to_cpu(hdr->len);

        switch (code & 0xff) {
        default:
                /* reply to a read command */
                if (hdr->priv != AR5523_CMD_ID) {
                        ar5523_err(ar, "Unexpected command id: %02x\n",
                                   code & 0xff);
                        goto skip;
                }
                ar5523_read_reply(ar, hdr, cmd);
                break;

        case WDCMSG_DEVICE_AVAIL:
                ar5523_dbg(ar, "WDCMSG_DEVICE_AVAIL\n");
                cmd->res = 0;
                cmd->olen = 0;
                complete(&cmd->done);
                break;

        case WDCMSG_SEND_COMPLETE:
                ar5523_dbg(ar, "WDCMSG_SEND_COMPLETE: %d pending\n",
                        atomic_read(&ar->tx_nr_pending));
                if (!test_bit(AR5523_HW_UP, &ar->flags))
                        ar5523_dbg(ar, "Unexpected WDCMSG_SEND_COMPLETE\n");
                else {
                        mod_timer(&ar->tx_wd_timer,
                                  jiffies + AR5523_TX_WD_TIMEOUT);
                        ar5523_data_tx_pkt_put(ar);

                }
                break;

        case WDCMSG_TARGET_START:
                /* This command returns a bogus id so it needs special
                   handling */
                dlen = hdrlen - sizeof(*hdr);
                if (dlen != (int)sizeof(u32)) {
                        ar5523_err(ar, "Invalid reply to WDCMSG_TARGET_START");
                        return;
                }
                if (!cmd->odata) {
                        ar5523_err(ar, "Unexpected WDCMSG_TARGET_START reply");
                        return;
                }
                memcpy(cmd->odata, hdr + 1, sizeof(u32));
                cmd->olen = sizeof(u32);
                cmd->res = 0;
                complete(&cmd->done);
                break;

        case WDCMSG_STATS_UPDATE:
                ar5523_dbg(ar, "WDCMSG_STATS_UPDATE\n");
                break;
        }

skip:
        ar5523_submit_rx_cmd(ar);
}

static int ar5523_alloc_rx_cmd(struct ar5523 *ar)
{
        ar->rx_cmd_urb = usb_alloc_urb(0, GFP_KERNEL);
        if (!ar->rx_cmd_urb)
                return -ENOMEM;

        ar->rx_cmd_buf = usb_alloc_coherent(ar->dev, AR5523_MAX_RXCMDSZ,
                                            GFP_KERNEL,
                                            &ar->rx_cmd_urb->transfer_dma);
        if (!ar->rx_cmd_buf) {
                usb_free_urb(ar->rx_cmd_urb);
                return -ENOMEM;
        }
        return 0;
}

static void ar5523_cancel_rx_cmd(struct ar5523 *ar)
{
        usb_kill_urb(ar->rx_cmd_urb);
}

static void ar5523_free_rx_cmd(struct ar5523 *ar)
{
        usb_free_coherent(ar->dev, AR5523_MAX_RXCMDSZ,
                          ar->rx_cmd_buf, ar->rx_cmd_urb->transfer_dma);
        usb_free_urb(ar->rx_cmd_urb);
}

static int ar5523_submit_rx_cmd(struct ar5523 *ar)
{
        int error;

        usb_fill_bulk_urb(ar->rx_cmd_urb, ar->dev,
                          ar5523_cmd_rx_pipe(ar->dev), ar->rx_cmd_buf,
                          AR5523_MAX_RXCMDSZ, ar5523_cmd_rx_cb, ar);
        ar->rx_cmd_urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;

        error = usb_submit_urb(ar->rx_cmd_urb, GFP_ATOMIC);
        if (error) {
                if (error != -ENODEV)
                        ar5523_err(ar, "error %d when submitting rx urb\n",
                                   error);
                return error;
        }
        return 0;
}

/*
 * Command submitted cb
 */
static void ar5523_cmd_tx_cb(struct urb *urb)
{
        struct ar5523_tx_cmd *cmd = urb->context;
        struct ar5523 *ar = cmd->ar;

        if (urb->status) {
                ar5523_err(ar, "Failed to TX command. Status = %d\n",
                           urb->status);
                cmd->res = urb->status;
                complete(&cmd->done);
                return;
        }

        if (!(cmd->flags & AR5523_CMD_FLAG_READ)) {
                cmd->res = 0;
                complete(&cmd->done);
        }
}

static void ar5523_cancel_tx_cmd(struct ar5523 *ar)
{
        usb_kill_urb(ar->tx_cmd.urb_tx);
}

static int ar5523_cmd(struct ar5523 *ar, u32 code, const void *idata,
                      int ilen, void *odata, int olen, int flags)
{
        struct ar5523_cmd_hdr *hdr;
        struct ar5523_tx_cmd *cmd = &ar->tx_cmd;
        int xferlen, error;

        /* always bulk-out a multiple of 4 bytes */
        xferlen = (sizeof(struct ar5523_cmd_hdr) + ilen + 3) & ~3;

        hdr = cmd->buf_tx;
        memset(hdr, 0, sizeof(struct ar5523_cmd_hdr));
        hdr->len  = cpu_to_be32(xferlen);
        hdr->code = cpu_to_be32(code);
        hdr->priv = AR5523_CMD_ID;

        if (flags & AR5523_CMD_FLAG_MAGIC)
                hdr->magic = cpu_to_be32(1 << 24);
        if (ilen)
                memcpy(hdr + 1, idata, ilen);

        cmd->odata = odata;
        cmd->olen = olen;
        cmd->flags = flags;

        ar5523_dbg(ar, "do cmd %02x\n", code);

        usb_fill_bulk_urb(cmd->urb_tx, ar->dev, ar5523_cmd_tx_pipe(ar->dev),
                          cmd->buf_tx, xferlen, ar5523_cmd_tx_cb, cmd);
        cmd->urb_tx->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;

        error = usb_submit_urb(cmd->urb_tx, GFP_KERNEL);
        if (error) {
                ar5523_err(ar, "could not send command 0x%x, error=%d\n",
                           code, error);
                return error;
        }

        if (!wait_for_completion_timeout(&cmd->done, 2 * HZ)) {
                ar5523_cancel_tx_cmd(ar);
                cmd->odata = NULL;
                ar5523_err(ar, "timeout waiting for command %02x reply\n",
                           code);
                cmd->res = -ETIMEDOUT;
        }
        return cmd->res;
}

static int ar5523_cmd_write(struct ar5523 *ar, u32 code, const void *data,
                            int len, int flags)
{
        flags &= ~AR5523_CMD_FLAG_READ;
        return ar5523_cmd(ar, code, data, len, NULL, 0, flags);
}

static int ar5523_cmd_read(struct ar5523 *ar, u32 code, const void *idata,
                           int ilen, void *odata, int olen, int flags)
{
        flags |= AR5523_CMD_FLAG_READ;
        return ar5523_cmd(ar, code, idata, ilen, odata, olen, flags);
}

static int ar5523_config(struct ar5523 *ar, u32 reg, u32 val)
{
        struct ar5523_write_mac write;
        int error;

        write.reg = cpu_to_be32(reg);
        write.len = cpu_to_be32(0);        /* 0 = single write */
        *(__be32 *)write.data = cpu_to_be32(val);

        error = ar5523_cmd_write(ar, WDCMSG_TARGET_SET_CONFIG, &write,
                                 3 * sizeof(u32), 0);
        if (error != 0)
                ar5523_err(ar, "could not write register 0x%02x\n", reg);
        return error;
}

static int ar5523_config_multi(struct ar5523 *ar, u32 reg, const void *data,
                               int len)
{
        struct ar5523_write_mac write;
        int error;

        write.reg = cpu_to_be32(reg);
        write.len = cpu_to_be32(len);
        memcpy(write.data, data, len);

        /* properly handle the case where len is zero (reset) */
        error = ar5523_cmd_write(ar, WDCMSG_TARGET_SET_CONFIG, &write,
            (len == 0) ? sizeof(u32) : 2 * sizeof(u32) + len, 0);
        if (error != 0)
                ar5523_err(ar, "could not write %d bytes to register 0x%02x\n",
                           len, reg);
        return error;
}

static int ar5523_get_status(struct ar5523 *ar, u32 which, void *odata,
                             int olen)
{
        int error;
        __be32 which_be;

        which_be = cpu_to_be32(which);
        error = ar5523_cmd_read(ar, WDCMSG_TARGET_GET_STATUS,
            &which_be, sizeof(which_be), odata, olen, AR5523_CMD_FLAG_MAGIC);
        if (error != 0)
                ar5523_err(ar, "could not read EEPROM offset 0x%02x\n", which);
        return error;
}

static int ar5523_get_capability(struct ar5523 *ar, u32 cap, u32 *val)
{
        int error;
        __be32 cap_be, val_be;

        cap_be = cpu_to_be32(cap);
        error = ar5523_cmd_read(ar, WDCMSG_TARGET_GET_CAPABILITY, &cap_be,
                                sizeof(cap_be), &val_be, sizeof(__be32),
                                AR5523_CMD_FLAG_MAGIC);
        if (error != 0) {
                ar5523_err(ar, "could not read capability %u\n", cap);
                return error;
        }
        *val = be32_to_cpu(val_be);
        return error;
}

static int ar5523_get_devcap(struct ar5523 *ar)
{
#define        GETCAP(x) do {                                \
        error = ar5523_get_capability(ar, x, &cap);                \
        if (error != 0)                                        \
                return error;                                \
        ar5523_info(ar, "Cap: "                        \
            "%s=0x%08x\n", #x, cap);        \
} while (0)
        int error;
        u32 cap;

        /* collect device capabilities */
        GETCAP(CAP_TARGET_VERSION);
        GETCAP(CAP_TARGET_REVISION);
        GETCAP(CAP_MAC_VERSION);
        GETCAP(CAP_MAC_REVISION);
        GETCAP(CAP_PHY_REVISION);
        GETCAP(CAP_ANALOG_5GHz_REVISION);
        GETCAP(CAP_ANALOG_2GHz_REVISION);

        GETCAP(CAP_REG_DOMAIN);
        GETCAP(CAP_REG_CAP_BITS);
        GETCAP(CAP_WIRELESS_MODES);
        GETCAP(CAP_CHAN_SPREAD_SUPPORT);
        GETCAP(CAP_COMPRESS_SUPPORT);
        GETCAP(CAP_BURST_SUPPORT);
        GETCAP(CAP_FAST_FRAMES_SUPPORT);
        GETCAP(CAP_CHAP_TUNING_SUPPORT);
        GETCAP(CAP_TURBOG_SUPPORT);
        GETCAP(CAP_TURBO_PRIME_SUPPORT);
        GETCAP(CAP_DEVICE_TYPE);
        GETCAP(CAP_WME_SUPPORT);
        GETCAP(CAP_TOTAL_QUEUES);
        GETCAP(CAP_CONNECTION_ID_MAX);

        GETCAP(CAP_LOW_5GHZ_CHAN);
        GETCAP(CAP_HIGH_5GHZ_CHAN);
        GETCAP(CAP_LOW_2GHZ_CHAN);
        GETCAP(CAP_HIGH_2GHZ_CHAN);
        GETCAP(CAP_TWICE_ANTENNAGAIN_5G);
        GETCAP(CAP_TWICE_ANTENNAGAIN_2G);

        GETCAP(CAP_CIPHER_AES_CCM);
        GETCAP(CAP_CIPHER_TKIP);
        GETCAP(CAP_MIC_TKIP);
        return 0;
}

static int ar5523_set_ledsteady(struct ar5523 *ar, int lednum, int ledmode)
{
        struct ar5523_cmd_ledsteady led;

        led.lednum = cpu_to_be32(lednum);
        led.ledmode = cpu_to_be32(ledmode);

        ar5523_dbg(ar, "set %s led %s (steady)\n",
                   (lednum == UATH_LED_LINK) ? "link" : "activity",
                   ledmode ? "on" : "off");
        return ar5523_cmd_write(ar, WDCMSG_SET_LED_STEADY, &led, sizeof(led),
                                 0);
}

static int ar5523_set_rxfilter(struct ar5523 *ar, u32 bits, u32 op)
{
        struct ar5523_cmd_rx_filter rxfilter;

        rxfilter.bits = cpu_to_be32(bits);
        rxfilter.op = cpu_to_be32(op);

        ar5523_dbg(ar, "setting Rx filter=0x%x flags=0x%x\n", bits, op);
        return ar5523_cmd_write(ar, WDCMSG_RX_FILTER, &rxfilter,
                                 sizeof(rxfilter), 0);
}

static int ar5523_reset_tx_queues(struct ar5523 *ar)
{
        __be32 qid = cpu_to_be32(0);

        ar5523_dbg(ar, "resetting Tx queue\n");
        return ar5523_cmd_write(ar, WDCMSG_RELEASE_TX_QUEUE,
                                 &qid, sizeof(qid), 0);
}

static int ar5523_set_chan(struct ar5523 *ar)
{
        struct ieee80211_conf *conf = &ar->hw->conf;

        struct ar5523_cmd_reset reset;

        memset(&reset, 0, sizeof(reset));
        reset.flags |= cpu_to_be32(UATH_CHAN_2GHZ);
        reset.flags |= cpu_to_be32(UATH_CHAN_OFDM);
        reset.freq = cpu_to_be32(conf->chandef.chan->center_freq);
        reset.maxrdpower = cpu_to_be32(50);        /* XXX */
        reset.channelchange = cpu_to_be32(1);
        reset.keeprccontent = cpu_to_be32(0);

        ar5523_dbg(ar, "set chan flags 0x%x freq %d\n",
                   be32_to_cpu(reset.flags),
                   conf->chandef.chan->center_freq);
        return ar5523_cmd_write(ar, WDCMSG_RESET, &reset, sizeof(reset), 0);
}

static int ar5523_queue_init(struct ar5523 *ar)
{
        struct ar5523_cmd_txq_setup qinfo;

        ar5523_dbg(ar, "setting up Tx queue\n");
        qinfo.qid             = cpu_to_be32(0);
        qinfo.len             = cpu_to_be32(sizeof(qinfo.attr));
        qinfo.attr.priority  = cpu_to_be32(0);        /* XXX */
        qinfo.attr.aifs             = cpu_to_be32(3);
        qinfo.attr.logcwmin  = cpu_to_be32(4);
        qinfo.attr.logcwmax  = cpu_to_be32(10);
        qinfo.attr.bursttime = cpu_to_be32(0);
        qinfo.attr.mode             = cpu_to_be32(0);
        qinfo.attr.qflags    = cpu_to_be32(1);        /* XXX? */
        return ar5523_cmd_write(ar, WDCMSG_SETUP_TX_QUEUE, &qinfo,
                                 sizeof(qinfo), 0);
}

static int ar5523_switch_chan(struct ar5523 *ar)
{
        int error;

        error = ar5523_set_chan(ar);
        if (error) {
                ar5523_err(ar, "could not set chan, error %d\n", error);
                goto out_err;
        }

        /* reset Tx rings */
        error = ar5523_reset_tx_queues(ar);
        if (error) {
                ar5523_err(ar, "could not reset Tx queues, error %d\n",
                           error);
                goto out_err;
        }
        /* set Tx rings WME properties */
        error = ar5523_queue_init(ar);
        if (error)
                ar5523_err(ar, "could not init wme, error %d\n", error);

out_err:
        return error;
}

static void ar5523_rx_data_put(struct ar5523 *ar,
                                struct ar5523_rx_data *data)
{
        unsigned long flags;
        spin_lock_irqsave(&ar->rx_data_list_lock, flags);
        list_move(&data->list, &ar->rx_data_free);
        spin_unlock_irqrestore(&ar->rx_data_list_lock, flags);
}

static void ar5523_data_rx_cb(struct urb *urb)
{
        struct ar5523_rx_data *data = urb->context;
        struct ar5523 *ar = data->ar;
        struct ar5523_rx_desc *desc;
        struct ar5523_chunk *chunk;
        struct ieee80211_hw *hw = ar->hw;
        struct ieee80211_rx_status *rx_status;
        u32 rxlen;
        int usblen = urb->actual_length;
        int hdrlen, pad;

        ar5523_dbg(ar, "%s\n", __func__);
        /* sync/async unlink faults aren't errors */
        if (urb->status) {
                if (urb->status != -ESHUTDOWN)
                        ar5523_err(ar, "%s: USB err: %d\n", __func__,
                                   urb->status);
                goto skip;
        }

        if (usblen < AR5523_MIN_RXBUFSZ) {
                ar5523_err(ar, "RX: wrong xfer size (usblen=%d)\n", usblen);
                goto skip;
        }

        chunk = (struct ar5523_chunk *) data->skb->data;

        if (((chunk->flags & UATH_CFLAGS_FINAL) == 0) ||
                chunk->seqnum != 0) {
                ar5523_dbg(ar, "RX: No final flag. s: %d f: %02x l: %d\n",
                           chunk->seqnum, chunk->flags,
                           be16_to_cpu(chunk->length));
                goto skip;
        }

        /* Rx descriptor is located at the end, 32-bit aligned */
        desc = (struct ar5523_rx_desc *)
                (data->skb->data + usblen - sizeof(struct ar5523_rx_desc));

        rxlen = be32_to_cpu(desc->len);
        if (rxlen > ar->rxbufsz) {
                ar5523_dbg(ar, "RX: Bad descriptor (len=%d)\n",
                           be32_to_cpu(desc->len));
                goto skip;
        }

        if (!rxlen) {
                ar5523_dbg(ar, "RX: rxlen is 0\n");
                goto skip;
        }

        if (be32_to_cpu(desc->status) != 0) {
                ar5523_dbg(ar, "Bad RX status (0x%x len = %d). Skip\n",
                           be32_to_cpu(desc->status), be32_to_cpu(desc->len));
                goto skip;
        }

        skb_reserve(data->skb, sizeof(*chunk));
        skb_put(data->skb, rxlen - sizeof(struct ar5523_rx_desc));

        hdrlen = ieee80211_get_hdrlen_from_skb(data->skb);
        if (!IS_ALIGNED(hdrlen, 4)) {
                ar5523_dbg(ar, "eek, alignment workaround activated\n");
                pad = ALIGN(hdrlen, 4) - hdrlen;
                memmove(data->skb->data + pad, data->skb->data, hdrlen);
                skb_pull(data->skb, pad);
                skb_put(data->skb, pad);
        }

        rx_status = IEEE80211_SKB_RXCB(data->skb);
        memset(rx_status, 0, sizeof(*rx_status));
        rx_status->freq = be32_to_cpu(desc->channel);
        rx_status->band = hw->conf.chandef.chan->band;
        rx_status->signal = -95 + be32_to_cpu(desc->rssi);

        ieee80211_rx_irqsafe(hw, data->skb);
        data->skb = NULL;

skip:
        if (data->skb) {
                dev_kfree_skb_irq(data->skb);
                data->skb = NULL;
        }

        ar5523_rx_data_put(ar, data);
        if (atomic_inc_return(&ar->rx_data_free_cnt) >=
            AR5523_RX_DATA_REFILL_COUNT &&
            test_bit(AR5523_HW_UP, &ar->flags))
                queue_work(ar->wq, &ar->rx_refill_work);
}

static void ar5523_rx_refill_work(struct work_struct *work)
{
        struct ar5523 *ar = container_of(work, struct ar5523, rx_refill_work);
        struct ar5523_rx_data *data;
        unsigned long flags;
        int error;

        ar5523_dbg(ar, "%s\n", __func__);
        do {
                spin_lock_irqsave(&ar->rx_data_list_lock, flags);

                if (!list_empty(&ar->rx_data_free))
                        data = (struct ar5523_rx_data *) ar->rx_data_free.next;
                else
                        data = NULL;
                spin_unlock_irqrestore(&ar->rx_data_list_lock, flags);

                if (!data)
                        goto done;

                data->skb = alloc_skb(ar->rxbufsz, GFP_KERNEL);
                if (!data->skb) {
                        ar5523_err(ar, "could not allocate rx skbuff\n");
                        return;
                }

                usb_fill_bulk_urb(data->urb, ar->dev,
                                  ar5523_data_rx_pipe(ar->dev), data->skb->data,
                                  ar->rxbufsz, ar5523_data_rx_cb, data);

                spin_lock_irqsave(&ar->rx_data_list_lock, flags);
                list_move(&data->list, &ar->rx_data_used);
                spin_unlock_irqrestore(&ar->rx_data_list_lock, flags);
                atomic_dec(&ar->rx_data_free_cnt);

                error = usb_submit_urb(data->urb, GFP_KERNEL);
                if (error) {
                        kfree_skb(data->skb);
                        if (error != -ENODEV)
                                ar5523_err(ar, "Err sending rx data urb %d\n",
                                           error);
                        ar5523_rx_data_put(ar, data);
                        atomic_inc(&ar->rx_data_free_cnt);
                        return;
                }

        } while (true);
done:
        return;
}

static void ar5523_cancel_rx_bufs(struct ar5523 *ar)
{
        struct ar5523_rx_data *data;
        unsigned long flags;

        do {
                spin_lock_irqsave(&ar->rx_data_list_lock, flags);
                if (!list_empty(&ar->rx_data_used))
                        data = (struct ar5523_rx_data *) ar->rx_data_used.next;
                else
                        data = NULL;
                spin_unlock_irqrestore(&ar->rx_data_list_lock, flags);

                if (!data)
                        break;

                usb_kill_urb(data->urb);
                list_move(&data->list, &ar->rx_data_free);
                atomic_inc(&ar->rx_data_free_cnt);
        } while (data);
}

static void ar5523_free_rx_bufs(struct ar5523 *ar)
{
        struct ar5523_rx_data *data;

        ar5523_cancel_rx_bufs(ar);
        while (!list_empty(&ar->rx_data_free)) {
                data = (struct ar5523_rx_data *) ar->rx_data_free.next;
                list_del(&data->list);
                usb_free_urb(data->urb);
        }
}

static int ar5523_alloc_rx_bufs(struct ar5523 *ar)
{
        int i;

        for (i = 0; i < AR5523_RX_DATA_COUNT; i++) {
                struct ar5523_rx_data *data = &ar->rx_data[i];

                data->ar = ar;
                data->urb = usb_alloc_urb(0, GFP_KERNEL);
                if (!data->urb)
                        goto err;
                list_add_tail(&data->list, &ar->rx_data_free);
                atomic_inc(&ar->rx_data_free_cnt);
        }
        return 0;

err:
        ar5523_free_rx_bufs(ar);
        return -ENOMEM;
}

static void ar5523_data_tx_pkt_put(struct ar5523 *ar)
{
        atomic_dec(&ar->tx_nr_total);
        if (!atomic_dec_return(&ar->tx_nr_pending)) {
                timer_delete(&ar->tx_wd_timer);
                wake_up(&ar->tx_flush_waitq);
        }

        if (atomic_read(&ar->tx_nr_total) < AR5523_TX_DATA_RESTART_COUNT) {
                ar5523_dbg(ar, "restart tx queue\n");
                ieee80211_wake_queues(ar->hw);
        }
}

static void ar5523_data_tx_cb(struct urb *urb)
{
        struct sk_buff *skb = urb->context;
        struct ieee80211_tx_info *txi = IEEE80211_SKB_CB(skb);
        struct ar5523_tx_data *data = (struct ar5523_tx_data *)
                                       txi->driver_data;
        struct ar5523 *ar = data->ar;
        unsigned long flags;

        ar5523_dbg(ar, "data tx urb completed: %d\n", urb->status);

        spin_lock_irqsave(&ar->tx_data_list_lock, flags);
        list_del(&data->list);
        spin_unlock_irqrestore(&ar->tx_data_list_lock, flags);

        if (urb->status) {
                ar5523_dbg(ar, "%s: urb status: %d\n", __func__, urb->status);
                ar5523_data_tx_pkt_put(ar);
                ieee80211_free_txskb(ar->hw, skb);
        } else {
                skb_pull(skb, sizeof(struct ar5523_tx_desc) + sizeof(__be32));
                ieee80211_tx_status_irqsafe(ar->hw, skb);
        }
        usb_free_urb(urb);
}

static void ar5523_tx(struct ieee80211_hw *hw,
                       struct ieee80211_tx_control *control,
                       struct sk_buff *skb)
{
        struct ieee80211_tx_info *txi = IEEE80211_SKB_CB(skb);
        struct ar5523_tx_data *data = (struct ar5523_tx_data *)
                                        txi->driver_data;
        struct ar5523 *ar = hw->priv;
        unsigned long flags;

        ar5523_dbg(ar, "tx called\n");
        if (atomic_inc_return(&ar->tx_nr_total) >= AR5523_TX_DATA_COUNT) {
                ar5523_dbg(ar, "tx queue full\n");
                ar5523_dbg(ar, "stop queues (tot %d pend %d)\n",
                           atomic_read(&ar->tx_nr_total),
                           atomic_read(&ar->tx_nr_pending));
                ieee80211_stop_queues(hw);
        }

        spin_lock_irqsave(&ar->tx_data_list_lock, flags);
        list_add_tail(&data->list, &ar->tx_queue_pending);
        spin_unlock_irqrestore(&ar->tx_data_list_lock, flags);

        ieee80211_queue_work(ar->hw, &ar->tx_work);
}

static void ar5523_tx_work_locked(struct ar5523 *ar)
{
        struct ar5523_tx_data *data;
        struct ar5523_tx_desc *desc;
        struct ar5523_chunk *chunk;
        struct ieee80211_tx_info *txi;
        struct urb *urb;
        struct sk_buff *skb;
        int error = 0, paylen;
        u32 txqid;
        unsigned long flags;

        BUILD_BUG_ON(sizeof(struct ar5523_tx_data) >
                     IEEE80211_TX_INFO_DRIVER_DATA_SIZE);

        ar5523_dbg(ar, "%s\n", __func__);
        do {
                spin_lock_irqsave(&ar->tx_data_list_lock, flags);
                if (!list_empty(&ar->tx_queue_pending)) {
                        data = (struct ar5523_tx_data *)
                                ar->tx_queue_pending.next;
                        list_del(&data->list);
                } else
                        data = NULL;
                spin_unlock_irqrestore(&ar->tx_data_list_lock, flags);

                if (!data)
                        break;

                txi = container_of((void *)data, struct ieee80211_tx_info,
                                   driver_data);
                txqid = 0;

                skb = container_of((void *)txi, struct sk_buff, cb);
                paylen = skb->len;

                urb = usb_alloc_urb(0, GFP_KERNEL);
                if (!urb) {
                        ieee80211_free_txskb(ar->hw, skb);
                        continue;
                }

                data->ar = ar;
                data->urb = urb;

                desc = skb_push(skb, sizeof(*desc));
                chunk = skb_push(skb, sizeof(*chunk));

                chunk->seqnum = 0;
                chunk->flags = UATH_CFLAGS_FINAL;
                chunk->length = cpu_to_be16(skb->len);

                desc->msglen = cpu_to_be32(skb->len);
                desc->msgid  = AR5523_DATA_ID;
                desc->buflen = cpu_to_be32(paylen);
                desc->type   = cpu_to_be32(WDCMSG_SEND);
                desc->flags  = cpu_to_be32(UATH_TX_NOTIFY);

                if (test_bit(AR5523_CONNECTED, &ar->flags))
                        desc->connid = cpu_to_be32(AR5523_ID_BSS);
                else
                        desc->connid = cpu_to_be32(AR5523_ID_BROADCAST);

                if (txi->flags & IEEE80211_TX_CTL_USE_MINRATE)
                        txqid |= UATH_TXQID_MINRATE;

                desc->txqid = cpu_to_be32(txqid);

                urb->transfer_flags = URB_ZERO_PACKET;
                usb_fill_bulk_urb(urb, ar->dev, ar5523_data_tx_pipe(ar->dev),
                                  skb->data, skb->len, ar5523_data_tx_cb, skb);

                spin_lock_irqsave(&ar->tx_data_list_lock, flags);
                list_add_tail(&data->list, &ar->tx_queue_submitted);
                spin_unlock_irqrestore(&ar->tx_data_list_lock, flags);
                mod_timer(&ar->tx_wd_timer, jiffies + AR5523_TX_WD_TIMEOUT);
                atomic_inc(&ar->tx_nr_pending);

                ar5523_dbg(ar, "TX Frame (%d pending)\n",
                           atomic_read(&ar->tx_nr_pending));
                error = usb_submit_urb(urb, GFP_KERNEL);
                if (error) {
                        ar5523_err(ar, "error %d when submitting tx urb\n",
                                   error);
                        spin_lock_irqsave(&ar->tx_data_list_lock, flags);
                        list_del(&data->list);
                        spin_unlock_irqrestore(&ar->tx_data_list_lock, flags);
                        atomic_dec(&ar->tx_nr_pending);
                        ar5523_data_tx_pkt_put(ar);
                        usb_free_urb(urb);
                        ieee80211_free_txskb(ar->hw, skb);
                }
        } while (true);
}

static void ar5523_tx_work(struct work_struct *work)
{
        struct ar5523 *ar = container_of(work, struct ar5523, tx_work);

        ar5523_dbg(ar, "%s\n", __func__);
        mutex_lock(&ar->mutex);
        ar5523_tx_work_locked(ar);
        mutex_unlock(&ar->mutex);
}

static void ar5523_tx_wd_timer(struct timer_list *t)
{
        struct ar5523 *ar = timer_container_of(ar, t, tx_wd_timer);

        ar5523_dbg(ar, "TX watchdog timer triggered\n");
        ieee80211_queue_work(ar->hw, &ar->tx_wd_work);
}

static void ar5523_tx_wd_work(struct work_struct *work)
{
        struct ar5523 *ar = container_of(work, struct ar5523, tx_wd_work);

        /* Occasionally the TX queues stop responding. The only way to
         * recover seems to be to reset the dongle.
         */

        mutex_lock(&ar->mutex);
        ar5523_err(ar, "TX queue stuck (tot %d pend %d)\n",
                   atomic_read(&ar->tx_nr_total),
                   atomic_read(&ar->tx_nr_pending));

        ar5523_err(ar, "Will restart dongle.\n");
        ar5523_cmd_write(ar, WDCMSG_TARGET_RESET, NULL, 0, 0);
        mutex_unlock(&ar->mutex);
}

static void ar5523_flush_tx(struct ar5523 *ar)
{
        ar5523_tx_work_locked(ar);

        /* Don't waste time trying to flush if USB is disconnected */
        if (test_bit(AR5523_USB_DISCONNECTED, &ar->flags))
                return;
        if (!wait_event_timeout(ar->tx_flush_waitq,
            !atomic_read(&ar->tx_nr_pending), AR5523_FLUSH_TIMEOUT))
                ar5523_err(ar, "flush timeout (tot %d pend %d)\n",
                           atomic_read(&ar->tx_nr_total),
                           atomic_read(&ar->tx_nr_pending));
}

static void ar5523_free_tx_cmd(struct ar5523 *ar)
{
        struct ar5523_tx_cmd *cmd = &ar->tx_cmd;

        usb_free_coherent(ar->dev, AR5523_MAX_RXCMDSZ, cmd->buf_tx,
                          cmd->urb_tx->transfer_dma);
        usb_free_urb(cmd->urb_tx);
}

static int ar5523_alloc_tx_cmd(struct ar5523 *ar)
{
        struct ar5523_tx_cmd *cmd = &ar->tx_cmd;

        cmd->ar = ar;
        init_completion(&cmd->done);

        cmd->urb_tx = usb_alloc_urb(0, GFP_KERNEL);
        if (!cmd->urb_tx)
                return -ENOMEM;
        cmd->buf_tx = usb_alloc_coherent(ar->dev, AR5523_MAX_TXCMDSZ,
                                         GFP_KERNEL,
                                         &cmd->urb_tx->transfer_dma);
        if (!cmd->buf_tx) {
                usb_free_urb(cmd->urb_tx);
                return -ENOMEM;
        }
        return 0;
}

/*
 * This function is called periodically (every second) when associated to
 * query device statistics.
 */
static void ar5523_stat_work(struct work_struct *work)
{
        struct ar5523 *ar = container_of(work, struct ar5523, stat_work.work);
        int error;

        ar5523_dbg(ar, "%s\n", __func__);
        mutex_lock(&ar->mutex);

        /*
         * Send request for statistics asynchronously once a second. This
         * seems to be important. Throughput is a lot better if this is done.
         */
        error = ar5523_cmd_write(ar, WDCMSG_TARGET_GET_STATS, NULL, 0, 0);
        if (error)
                ar5523_err(ar, "could not query stats, error %d\n", error);
        mutex_unlock(&ar->mutex);
        ieee80211_queue_delayed_work(ar->hw, &ar->stat_work, HZ);
}

/*
 * Interface routines to the mac80211 stack.
 */
static int ar5523_start(struct ieee80211_hw *hw)
{
        struct ar5523 *ar = hw->priv;
        int error;
        __be32 val;

        ar5523_dbg(ar, "start called\n");

        mutex_lock(&ar->mutex);
        val = cpu_to_be32(0);
        ar5523_cmd_write(ar, WDCMSG_BIND, &val, sizeof(val), 0);

        /* set MAC address */
        ar5523_config_multi(ar, CFG_MAC_ADDR, &ar->hw->wiphy->perm_addr,
                            ETH_ALEN);

        /* XXX honor net80211 state */
        ar5523_config(ar, CFG_RATE_CONTROL_ENABLE, 0x00000001);
        ar5523_config(ar, CFG_DIVERSITY_CTL, 0x00000001);
        ar5523_config(ar, CFG_ABOLT, 0x0000003f);
        ar5523_config(ar, CFG_WME_ENABLED, 0x00000000);

        ar5523_config(ar, CFG_SERVICE_TYPE, 1);
        ar5523_config(ar, CFG_TP_SCALE, 0x00000000);
        ar5523_config(ar, CFG_TPC_HALF_DBM5, 0x0000003c);
        ar5523_config(ar, CFG_TPC_HALF_DBM2, 0x0000003c);
        ar5523_config(ar, CFG_OVERRD_TX_POWER, 0x00000000);
        ar5523_config(ar, CFG_GMODE_PROTECTION, 0x00000000);
        ar5523_config(ar, CFG_GMODE_PROTECT_RATE_INDEX, 0x00000003);
        ar5523_config(ar, CFG_PROTECTION_TYPE, 0x00000000);
        ar5523_config(ar, CFG_MODE_CTS, 0x00000002);

        error = ar5523_cmd_read(ar, WDCMSG_TARGET_START, NULL, 0,
            &val, sizeof(val), AR5523_CMD_FLAG_MAGIC);
        if (error) {
                ar5523_dbg(ar, "could not start target, error %d\n", error);
                goto err;
        }
        ar5523_dbg(ar, "WDCMSG_TARGET_START returns handle: 0x%x\n",
                   be32_to_cpu(val));

        ar5523_switch_chan(ar);

        val = cpu_to_be32(TARGET_DEVICE_AWAKE);
        ar5523_cmd_write(ar, WDCMSG_SET_PWR_MODE, &val, sizeof(val), 0);
        /* XXX? check */
        ar5523_cmd_write(ar, WDCMSG_RESET_KEY_CACHE, NULL, 0, 0);

        set_bit(AR5523_HW_UP, &ar->flags);
        queue_work(ar->wq, &ar->rx_refill_work);

        /* enable Rx */
        ar5523_set_rxfilter(ar, 0, UATH_FILTER_OP_INIT);
        ar5523_set_rxfilter(ar,
                            UATH_FILTER_RX_UCAST | UATH_FILTER_RX_MCAST |
                            UATH_FILTER_RX_BCAST | UATH_FILTER_RX_BEACON,
                            UATH_FILTER_OP_SET);

        ar5523_set_ledsteady(ar, UATH_LED_ACTIVITY, UATH_LED_ON);
        ar5523_dbg(ar, "start OK\n");

err:
        mutex_unlock(&ar->mutex);
        return error;
}

static void ar5523_stop(struct ieee80211_hw *hw, bool suspend)
{
        struct ar5523 *ar = hw->priv;

        ar5523_dbg(ar, "stop called\n");

        cancel_delayed_work_sync(&ar->stat_work);
        mutex_lock(&ar->mutex);
        clear_bit(AR5523_HW_UP, &ar->flags);

        ar5523_set_ledsteady(ar, UATH_LED_LINK, UATH_LED_OFF);
        ar5523_set_ledsteady(ar, UATH_LED_ACTIVITY, UATH_LED_OFF);

        ar5523_cmd_write(ar, WDCMSG_TARGET_STOP, NULL, 0, 0);

        timer_delete_sync(&ar->tx_wd_timer);
        cancel_work_sync(&ar->tx_wd_work);
        cancel_work_sync(&ar->rx_refill_work);
        ar5523_cancel_rx_bufs(ar);
        mutex_unlock(&ar->mutex);
}

static int ar5523_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
                                    u32 value)
{
        struct ar5523 *ar = hw->priv;
        int ret;

        ar5523_dbg(ar, "set_rts_threshold called\n");
        mutex_lock(&ar->mutex);

        ret = ar5523_config(ar, CFG_USER_RTS_THRESHOLD, value);

        mutex_unlock(&ar->mutex);
        return ret;
}

static void ar5523_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
                         u32 queues, bool drop)
{
        struct ar5523 *ar = hw->priv;

        ar5523_dbg(ar, "flush called\n");
        ar5523_flush_tx(ar);
}

static int ar5523_add_interface(struct ieee80211_hw *hw,
                                struct ieee80211_vif *vif)
{
        struct ar5523 *ar = hw->priv;

        ar5523_dbg(ar, "add interface called\n");

        if (ar->vif) {
                ar5523_dbg(ar, "invalid add_interface\n");
                return -EOPNOTSUPP;
        }

        switch (vif->type) {
        case NL80211_IFTYPE_STATION:
                ar->vif = vif;
                break;
        default:
                return -EOPNOTSUPP;
        }
        return 0;
}

static void ar5523_remove_interface(struct ieee80211_hw *hw,
                                    struct ieee80211_vif *vif)
{
        struct ar5523 *ar = hw->priv;

        ar5523_dbg(ar, "remove interface called\n");
        ar->vif = NULL;
}

static int ar5523_hwconfig(struct ieee80211_hw *hw, int radio_idx, u32 changed)
{
        struct ar5523 *ar = hw->priv;

        ar5523_dbg(ar, "config called\n");
        mutex_lock(&ar->mutex);
        if (changed & IEEE80211_CONF_CHANGE_CHANNEL) {
                ar5523_dbg(ar, "Do channel switch\n");
                ar5523_flush_tx(ar);
                ar5523_switch_chan(ar);
        }
        mutex_unlock(&ar->mutex);
        return 0;
}

static int ar5523_get_wlan_mode(struct ar5523 *ar,
                                struct ieee80211_bss_conf *bss_conf)
{
        struct ieee80211_supported_band *band;
        int bit;
        struct ieee80211_sta *sta;
        u32 sta_rate_set;

        band = ar->hw->wiphy->bands[ar->hw->conf.chandef.chan->band];
        sta = ieee80211_find_sta(ar->vif, bss_conf->bssid);
        if (!sta) {
                ar5523_info(ar, "STA not found!\n");
                return WLAN_MODE_11b;
        }
        sta_rate_set = sta->deflink.supp_rates[ar->hw->conf.chandef.chan->band];

        for (bit = 0; bit < band->n_bitrates; bit++) {
                if (sta_rate_set & 1) {
                        int rate = band->bitrates[bit].bitrate;
                        switch (rate) {
                        case 60:
                        case 90:
                        case 120:
                        case 180:
                        case 240:
                        case 360:
                        case 480:
                        case 540:
                                return WLAN_MODE_11g;
                        }
                }
                sta_rate_set >>= 1;
        }
        return WLAN_MODE_11b;
}

static void ar5523_create_rateset(struct ar5523 *ar,
                                  struct ieee80211_bss_conf *bss_conf,
                                  struct ar5523_cmd_rateset *rs,
                                  bool basic)
{
        struct ieee80211_supported_band *band;
        struct ieee80211_sta *sta;
        int bit, i = 0;
        u32 sta_rate_set, basic_rate_set;

        sta = ieee80211_find_sta(ar->vif, bss_conf->bssid);
        basic_rate_set = bss_conf->basic_rates;
        if (!sta) {
                ar5523_info(ar, "STA not found. Cannot set rates\n");
                sta_rate_set = bss_conf->basic_rates;
        } else
                sta_rate_set = sta->deflink.supp_rates[ar->hw->conf.chandef.chan->band];

        ar5523_dbg(ar, "sta rate_set = %08x\n", sta_rate_set);

        band = ar->hw->wiphy->bands[ar->hw->conf.chandef.chan->band];
        for (bit = 0; bit < band->n_bitrates; bit++) {
                BUG_ON(i >= AR5523_MAX_NRATES);
                ar5523_dbg(ar, "Considering rate %d : %d\n",
                           band->bitrates[bit].hw_value, sta_rate_set & 1);
                if (sta_rate_set & 1) {
                        rs->set[i] = band->bitrates[bit].hw_value;
                        if (basic_rate_set & 1 && basic)
                                rs->set[i] |= 0x80;
                        i++;
                }
                sta_rate_set >>= 1;
                basic_rate_set >>= 1;
        }

        rs->length = i;
}

static int ar5523_set_basic_rates(struct ar5523 *ar,
                                  struct ieee80211_bss_conf *bss)
{
        struct ar5523_cmd_rates rates;

        memset(&rates, 0, sizeof(rates));
        rates.connid = cpu_to_be32(2);                /* XXX */
        rates.size   = cpu_to_be32(sizeof(struct ar5523_cmd_rateset));
        ar5523_create_rateset(ar, bss, &rates.rateset, true);

        return ar5523_cmd_write(ar, WDCMSG_SET_BASIC_RATE, &rates,
                                sizeof(rates), 0);
}

static int ar5523_create_connection(struct ar5523 *ar,
                                    struct ieee80211_vif *vif,
                                    struct ieee80211_bss_conf *bss)
{
        struct ar5523_cmd_create_connection create;
        int wlan_mode;

        memset(&create, 0, sizeof(create));
        create.connid = cpu_to_be32(2);
        create.bssid = cpu_to_be32(0);
        /* XXX packed or not?  */
        create.size = cpu_to_be32(sizeof(struct ar5523_cmd_rateset));

        ar5523_create_rateset(ar, bss, &create.connattr.rateset, false);

        wlan_mode = ar5523_get_wlan_mode(ar, bss);
        create.connattr.wlanmode = cpu_to_be32(wlan_mode);

        return ar5523_cmd_write(ar, WDCMSG_CREATE_CONNECTION, &create,
                                sizeof(create), 0);
}

static int ar5523_write_associd(struct ar5523 *ar, struct ieee80211_vif *vif)
{
        struct ieee80211_bss_conf *bss = &vif->bss_conf;
        struct ar5523_cmd_set_associd associd;

        memset(&associd, 0, sizeof(associd));
        associd.defaultrateix = cpu_to_be32(0);        /* XXX */
        associd.associd = cpu_to_be32(vif->cfg.aid);
        associd.timoffset = cpu_to_be32(0x3b);        /* XXX */
        memcpy(associd.bssid, bss->bssid, ETH_ALEN);
        return ar5523_cmd_write(ar, WDCMSG_WRITE_ASSOCID, &associd,
                                sizeof(associd), 0);
}

static void ar5523_bss_info_changed(struct ieee80211_hw *hw,
                                    struct ieee80211_vif *vif,
                                    struct ieee80211_bss_conf *bss,
                                    u64 changed)
{
        struct ar5523 *ar = hw->priv;
        int error;

        ar5523_dbg(ar, "bss_info_changed called\n");
        mutex_lock(&ar->mutex);

        if (!(changed & BSS_CHANGED_ASSOC))
                goto out_unlock;

        if (vif->cfg.assoc) {
                error = ar5523_create_connection(ar, vif, bss);
                if (error) {
                        ar5523_err(ar, "could not create connection\n");
                        goto out_unlock;
                }

                error = ar5523_set_basic_rates(ar, bss);
                if (error) {
                        ar5523_err(ar, "could not set negotiated rate set\n");
                        goto out_unlock;
                }

                error = ar5523_write_associd(ar, vif);
                if (error) {
                        ar5523_err(ar, "could not set association\n");
                        goto out_unlock;
                }

                /* turn link LED on */
                ar5523_set_ledsteady(ar, UATH_LED_LINK, UATH_LED_ON);
                set_bit(AR5523_CONNECTED, &ar->flags);
                ieee80211_queue_delayed_work(hw, &ar->stat_work, HZ);

        } else {
                cancel_delayed_work(&ar->stat_work);
                clear_bit(AR5523_CONNECTED, &ar->flags);
                ar5523_set_ledsteady(ar, UATH_LED_LINK, UATH_LED_OFF);
        }

out_unlock:
        mutex_unlock(&ar->mutex);

}

#define AR5523_SUPPORTED_FILTERS (FIF_ALLMULTI | \
                                  FIF_FCSFAIL | \
                                  FIF_OTHER_BSS)

static void ar5523_configure_filter(struct ieee80211_hw *hw,
                                    unsigned int changed_flags,
                                    unsigned int *total_flags,
                                    u64 multicast)
{
        struct ar5523 *ar = hw->priv;
        u32 filter = 0;

        ar5523_dbg(ar, "configure_filter called\n");
        mutex_lock(&ar->mutex);
        ar5523_flush_tx(ar);

        *total_flags &= AR5523_SUPPORTED_FILTERS;

        /* The filters seems strange. UATH_FILTER_RX_BCAST and
         * UATH_FILTER_RX_MCAST does not result in those frames being RXed.
         * The only way I have found to get [mb]cast frames seems to be
         * to set UATH_FILTER_RX_PROM. */
        filter |= UATH_FILTER_RX_UCAST | UATH_FILTER_RX_MCAST |
                  UATH_FILTER_RX_BCAST | UATH_FILTER_RX_BEACON |
                  UATH_FILTER_RX_PROM;

        ar5523_set_rxfilter(ar, 0, UATH_FILTER_OP_INIT);
        ar5523_set_rxfilter(ar, filter, UATH_FILTER_OP_SET);

        mutex_unlock(&ar->mutex);
}

static const struct ieee80211_ops ar5523_ops = {
        .add_chanctx = ieee80211_emulate_add_chanctx,
        .remove_chanctx = ieee80211_emulate_remove_chanctx,
        .change_chanctx = ieee80211_emulate_change_chanctx,
        .switch_vif_chanctx = ieee80211_emulate_switch_vif_chanctx,
        .start                        = ar5523_start,
        .stop                        = ar5523_stop,
        .tx                        = ar5523_tx,
        .wake_tx_queue                = ieee80211_handle_wake_tx_queue,
        .set_rts_threshold        = ar5523_set_rts_threshold,
        .add_interface                = ar5523_add_interface,
        .remove_interface        = ar5523_remove_interface,
        .config                        = ar5523_hwconfig,
        .bss_info_changed        = ar5523_bss_info_changed,
        .configure_filter        = ar5523_configure_filter,
        .flush                        = ar5523_flush,
};

static int ar5523_host_available(struct ar5523 *ar)
{
        struct ar5523_cmd_host_available setup;

        /* inform target the host is available */
        setup.sw_ver_major = cpu_to_be32(ATH_SW_VER_MAJOR);
        setup.sw_ver_minor = cpu_to_be32(ATH_SW_VER_MINOR);
        setup.sw_ver_patch = cpu_to_be32(ATH_SW_VER_PATCH);
        setup.sw_ver_build = cpu_to_be32(ATH_SW_VER_BUILD);
        return ar5523_cmd_read(ar, WDCMSG_HOST_AVAILABLE,
                               &setup, sizeof(setup), NULL, 0, 0);
}

static int ar5523_get_devstatus(struct ar5523 *ar)
{
        u8 macaddr[ETH_ALEN];
        int error;

        /* retrieve MAC address */
        error = ar5523_get_status(ar, ST_MAC_ADDR, macaddr, ETH_ALEN);
        if (error) {
                ar5523_err(ar, "could not read MAC address\n");
                return error;
        }

        SET_IEEE80211_PERM_ADDR(ar->hw, macaddr);

        error = ar5523_get_status(ar, ST_SERIAL_NUMBER,
            &ar->serial[0], sizeof(ar->serial));
        if (error) {
                ar5523_err(ar, "could not read device serial number\n");
                return error;
        }
        return 0;
}

#define AR5523_SANE_RXBUFSZ 2000

static int ar5523_get_max_rxsz(struct ar5523 *ar)
{
        int error;
        __be32 rxsize;

        /* Get max rx size */
        error = ar5523_get_status(ar, ST_WDC_TRANSPORT_CHUNK_SIZE, &rxsize,
                                  sizeof(rxsize));
        if (error != 0) {
                ar5523_err(ar, "could not read max RX size\n");
                return error;
        }

        ar->rxbufsz = be32_to_cpu(rxsize);

        if (!ar->rxbufsz || ar->rxbufsz > AR5523_SANE_RXBUFSZ) {
                ar5523_err(ar, "Bad rxbufsz from device. Using %d instead\n",
                           AR5523_SANE_RXBUFSZ);
                ar->rxbufsz = AR5523_SANE_RXBUFSZ;
        }

        ar5523_dbg(ar, "Max RX buf size: %d\n", ar->rxbufsz);
        return 0;
}

/*
 * This is copied from rtl818x, but we should probably move this
 * to common code as in OpenBSD.
 */
static const struct ieee80211_rate ar5523_rates[] = {
        { .bitrate = 10, .hw_value = 2, },
        { .bitrate = 20, .hw_value = 4 },
        { .bitrate = 55, .hw_value = 11, },
        { .bitrate = 110, .hw_value = 22, },
        { .bitrate = 60, .hw_value = 12, },
        { .bitrate = 90, .hw_value = 18, },
        { .bitrate = 120, .hw_value = 24, },
        { .bitrate = 180, .hw_value = 36, },
        { .bitrate = 240, .hw_value = 48, },
        { .bitrate = 360, .hw_value = 72, },
        { .bitrate = 480, .hw_value = 96, },
        { .bitrate = 540, .hw_value = 108, },
};

static const struct ieee80211_channel ar5523_channels[] = {
        { .center_freq = 2412 },
        { .center_freq = 2417 },
        { .center_freq = 2422 },
        { .center_freq = 2427 },
        { .center_freq = 2432 },
        { .center_freq = 2437 },
        { .center_freq = 2442 },
        { .center_freq = 2447 },
        { .center_freq = 2452 },
        { .center_freq = 2457 },
        { .center_freq = 2462 },
        { .center_freq = 2467 },
        { .center_freq = 2472 },
        { .center_freq = 2484 },
};

static int ar5523_init_modes(struct ar5523 *ar)
{
        BUILD_BUG_ON(sizeof(ar->channels) != sizeof(ar5523_channels));
        BUILD_BUG_ON(sizeof(ar->rates) != sizeof(ar5523_rates));

        memcpy(ar->channels, ar5523_channels, sizeof(ar5523_channels));
        memcpy(ar->rates, ar5523_rates, sizeof(ar5523_rates));

        ar->band.band = NL80211_BAND_2GHZ;
        ar->band.channels = ar->channels;
        ar->band.n_channels = ARRAY_SIZE(ar5523_channels);
        ar->band.bitrates = ar->rates;
        ar->band.n_bitrates = ARRAY_SIZE(ar5523_rates);
        ar->hw->wiphy->bands[NL80211_BAND_2GHZ] = &ar->band;
        return 0;
}

/*
 * Load the MIPS R4000 microcode into the device.  Once the image is loaded,
 * the device will detach itself from the bus and reattach later with a new
 * product Id (a la ezusb).
 */
static int ar5523_load_firmware(struct usb_device *dev)
{
        struct ar5523_fwblock *txblock, *rxblock;
        const struct firmware *fw;
        void *fwbuf;
        int len, offset;
        int foolen; /* XXX(hch): handle short transfers */
        int error = -ENXIO;

        if (request_firmware(&fw, AR5523_FIRMWARE_FILE, &dev->dev)) {
                dev_err(&dev->dev, "no firmware found: %s\n",
                        AR5523_FIRMWARE_FILE);
                return -ENOENT;
        }

        txblock = kzalloc(sizeof(*txblock), GFP_KERNEL);
        if (!txblock)
                goto out;

        rxblock = kmalloc(sizeof(*rxblock), GFP_KERNEL);
        if (!rxblock)
                goto out_free_txblock;

        fwbuf = kmalloc(AR5523_MAX_FWBLOCK_SIZE, GFP_KERNEL);
        if (!fwbuf)
                goto out_free_rxblock;

        txblock->flags = cpu_to_be32(AR5523_WRITE_BLOCK);
        txblock->total = cpu_to_be32(fw->size);

        offset = 0;
        len = fw->size;
        while (len > 0) {
                int mlen = min(len, AR5523_MAX_FWBLOCK_SIZE);

                txblock->remain = cpu_to_be32(len - mlen);
                txblock->len = cpu_to_be32(mlen);

                /* send firmware block meta-data */
                error = usb_bulk_msg(dev, ar5523_cmd_tx_pipe(dev),
                                     txblock, sizeof(*txblock), &foolen,
                                     AR5523_CMD_TIMEOUT);
                if (error) {
                        dev_err(&dev->dev,
                                "could not send firmware block info\n");
                        goto out_free_fwbuf;
                }

                /* send firmware block data */
                memcpy(fwbuf, fw->data + offset, mlen);
                error = usb_bulk_msg(dev, ar5523_data_tx_pipe(dev),
                                     fwbuf, mlen, &foolen,
                                     AR5523_DATA_TIMEOUT);
                if (error) {
                        dev_err(&dev->dev,
                                "could not send firmware block data\n");
                        goto out_free_fwbuf;
                }

                /* wait for ack from firmware */
                error = usb_bulk_msg(dev, ar5523_cmd_rx_pipe(dev),
                                     rxblock, sizeof(*rxblock), &foolen,
                                     AR5523_CMD_TIMEOUT);
                if (error) {
                        dev_err(&dev->dev,
                                "could not read firmware answer\n");
                        goto out_free_fwbuf;
                }

                len -= mlen;
                offset += mlen;
        }

        /*
         * Set the error to -ENXIO to make sure we continue probing for
         * a driver.
         */
        error = -ENXIO;

 out_free_fwbuf:
        kfree(fwbuf);
 out_free_rxblock:
        kfree(rxblock);
 out_free_txblock:
        kfree(txblock);
 out:
        release_firmware(fw);
        return error;
}

static int ar5523_probe(struct usb_interface *intf,
                        const struct usb_device_id *id)
{
        struct usb_device *dev = interface_to_usbdev(intf);
        struct ieee80211_hw *hw;
        struct ar5523 *ar;
        int error = -ENOMEM;

        static const u8 bulk_ep_addr[] = {
                AR5523_CMD_TX_PIPE | USB_DIR_OUT,
                AR5523_DATA_TX_PIPE | USB_DIR_OUT,
                AR5523_CMD_RX_PIPE | USB_DIR_IN,
                AR5523_DATA_RX_PIPE | USB_DIR_IN,
                0};

        if (!usb_check_bulk_endpoints(intf, bulk_ep_addr)) {
                dev_err(&dev->dev,
                        "Could not find all expected endpoints\n");
                error = -ENODEV;
                goto out;
        }

        /*
         * Load firmware if the device requires it.  This will return
         * -ENXIO on success and we'll get called back afer the usb
         * id changes to indicate that the firmware is present.
         */
        if (id->driver_info & AR5523_FLAG_PRE_FIRMWARE)
                return ar5523_load_firmware(dev);


        hw = ieee80211_alloc_hw(sizeof(*ar), &ar5523_ops);
        if (!hw)
                goto out;
        SET_IEEE80211_DEV(hw, &intf->dev);

        ar = hw->priv;
        ar->hw = hw;
        ar->dev = dev;
        mutex_init(&ar->mutex);

        INIT_DELAYED_WORK(&ar->stat_work, ar5523_stat_work);
        timer_setup(&ar->tx_wd_timer, ar5523_tx_wd_timer, 0);
        INIT_WORK(&ar->tx_wd_work, ar5523_tx_wd_work);
        INIT_WORK(&ar->tx_work, ar5523_tx_work);
        INIT_LIST_HEAD(&ar->tx_queue_pending);
        INIT_LIST_HEAD(&ar->tx_queue_submitted);
        spin_lock_init(&ar->tx_data_list_lock);
        atomic_set(&ar->tx_nr_total, 0);
        atomic_set(&ar->tx_nr_pending, 0);
        init_waitqueue_head(&ar->tx_flush_waitq);

        atomic_set(&ar->rx_data_free_cnt, 0);
        INIT_WORK(&ar->rx_refill_work, ar5523_rx_refill_work);
        INIT_LIST_HEAD(&ar->rx_data_free);
        INIT_LIST_HEAD(&ar->rx_data_used);
        spin_lock_init(&ar->rx_data_list_lock);

        ar->wq = create_singlethread_workqueue("ar5523");
        if (!ar->wq) {
                ar5523_err(ar, "Could not create wq\n");
                goto out_free_ar;
        }

        error = ar5523_alloc_rx_bufs(ar);
        if (error) {
                ar5523_err(ar, "Could not allocate rx buffers\n");
                goto out_free_wq;
        }

        error = ar5523_alloc_rx_cmd(ar);
        if (error) {
                ar5523_err(ar, "Could not allocate rx command buffers\n");
                goto out_free_rx_bufs;
        }

        error = ar5523_alloc_tx_cmd(ar);
        if (error) {
                ar5523_err(ar, "Could not allocate tx command buffers\n");
                goto out_free_rx_cmd;
        }

        error = ar5523_submit_rx_cmd(ar);
        if (error) {
                ar5523_err(ar, "Failed to submit rx cmd\n");
                goto out_free_tx_cmd;
        }

        /*
         * We're now ready to send/receive firmware commands.
         */
        error = ar5523_host_available(ar);
        if (error) {
                ar5523_err(ar, "could not initialize adapter\n");
                goto out_cancel_rx_cmd;
        }

        error = ar5523_get_max_rxsz(ar);
        if (error) {
                ar5523_err(ar, "could not get caps from adapter\n");
                goto out_cancel_rx_cmd;
        }

        error = ar5523_get_devcap(ar);
        if (error) {
                ar5523_err(ar, "could not get caps from adapter\n");
                goto out_cancel_rx_cmd;
        }

        error = ar5523_get_devstatus(ar);
        if (error != 0) {
                ar5523_err(ar, "could not get device status\n");
                goto out_cancel_rx_cmd;
        }

        ar5523_info(ar, "MAC/BBP AR5523, RF AR%c112\n",
                        (id->driver_info & AR5523_FLAG_ABG) ? '5' : '2');

        ar->vif = NULL;
        ieee80211_hw_set(hw, HAS_RATE_CONTROL);
        ieee80211_hw_set(hw, RX_INCLUDES_FCS);
        ieee80211_hw_set(hw, SIGNAL_DBM);
        hw->extra_tx_headroom = sizeof(struct ar5523_tx_desc) +
                                sizeof(struct ar5523_chunk);
        hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION);
        hw->queues = 1;

        error = ar5523_init_modes(ar);
        if (error)
                goto out_cancel_rx_cmd;

        wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST);

        usb_set_intfdata(intf, hw);

        error = ieee80211_register_hw(hw);
        if (error) {
                ar5523_err(ar, "could not register device\n");
                goto out_cancel_rx_cmd;
        }

        ar5523_info(ar, "Found and initialized AR5523 device\n");
        return 0;

out_cancel_rx_cmd:
        ar5523_cancel_rx_cmd(ar);
out_free_tx_cmd:
        ar5523_free_tx_cmd(ar);
out_free_rx_cmd:
        ar5523_free_rx_cmd(ar);
out_free_rx_bufs:
        ar5523_free_rx_bufs(ar);
out_free_wq:
        destroy_workqueue(ar->wq);
out_free_ar:
        ieee80211_free_hw(hw);
out:
        return error;
}

static void ar5523_disconnect(struct usb_interface *intf)
{
        struct ieee80211_hw *hw = usb_get_intfdata(intf);
        struct ar5523 *ar = hw->priv;

        ar5523_dbg(ar, "detaching\n");
        set_bit(AR5523_USB_DISCONNECTED, &ar->flags);

        ieee80211_unregister_hw(hw);

        ar5523_cancel_rx_cmd(ar);
        ar5523_free_tx_cmd(ar);
        ar5523_free_rx_cmd(ar);
        ar5523_free_rx_bufs(ar);

        destroy_workqueue(ar->wq);

        ieee80211_free_hw(hw);
        usb_set_intfdata(intf, NULL);
}

#define AR5523_DEVICE_UG(vendor, device) \
        { USB_DEVICE((vendor), (device)) }, \
        { USB_DEVICE((vendor), (device) + 1), \
                .driver_info = AR5523_FLAG_PRE_FIRMWARE }
#define AR5523_DEVICE_UX(vendor, device) \
        { USB_DEVICE((vendor), (device)), \
                .driver_info = AR5523_FLAG_ABG }, \
        { USB_DEVICE((vendor), (device) + 1), \
                .driver_info = AR5523_FLAG_ABG|AR5523_FLAG_PRE_FIRMWARE }

static const struct usb_device_id ar5523_id_table[] = {
        AR5523_DEVICE_UG(0x168c, 0x0001),        /* Atheros / AR5523 */
        AR5523_DEVICE_UG(0x0cf3, 0x0001),        /* Atheros2 / AR5523_1 */
        AR5523_DEVICE_UG(0x0cf3, 0x0003),        /* Atheros2 / AR5523_2 */
        AR5523_DEVICE_UX(0x0cf3, 0x0005),        /* Atheros2 / AR5523_3 */
        AR5523_DEVICE_UG(0x0d8e, 0x7801),        /* Conceptronic / AR5523_1 */
        AR5523_DEVICE_UX(0x0d8e, 0x7811),        /* Conceptronic / AR5523_2 */
        AR5523_DEVICE_UX(0x2001, 0x3a00),        /* Dlink / DWLAG132 */
        AR5523_DEVICE_UG(0x2001, 0x3a02),        /* Dlink / DWLG132 */
        AR5523_DEVICE_UX(0x2001, 0x3a04),        /* Dlink / DWLAG122 */
        AR5523_DEVICE_UG(0x07d1, 0x3a07),        /* D-Link / WUA-2340 rev A1 */
        AR5523_DEVICE_UG(0x1690, 0x0712),        /* Gigaset / AR5523 */
        AR5523_DEVICE_UG(0x1690, 0x0710),        /* Gigaset / SMCWUSBTG */
        AR5523_DEVICE_UG(0x129b, 0x160b),        /* Gigaset / USB stick 108
                                                   (CyberTAN Technology) */
        AR5523_DEVICE_UG(0x16ab, 0x7801),        /* Globalsun / AR5523_1 */
        AR5523_DEVICE_UX(0x16ab, 0x7811),        /* Globalsun / AR5523_2 */
        AR5523_DEVICE_UG(0x0d8e, 0x7802),        /* Globalsun / AR5523_3 */
        AR5523_DEVICE_UX(0x0846, 0x4300),        /* Netgear / WG111U */
        AR5523_DEVICE_UG(0x0846, 0x4250),        /* Netgear / WG111T */
        AR5523_DEVICE_UG(0x0846, 0x5f00),        /* Netgear / WPN111 */
        AR5523_DEVICE_UG(0x083a, 0x4506),        /* SMC / EZ Connect
                                                   SMCWUSBT-G2 */
        AR5523_DEVICE_UG(0x157e, 0x3006),        /* Umedia / AR5523_1, TEW444UBEU*/
        AR5523_DEVICE_UX(0x157e, 0x3205),        /* Umedia / AR5523_2 */
        AR5523_DEVICE_UG(0x1435, 0x0826),        /* Wistronneweb / AR5523_1 */
        AR5523_DEVICE_UX(0x1435, 0x0828),        /* Wistronneweb / AR5523_2 */
        AR5523_DEVICE_UG(0x0cde, 0x0012),        /* Zcom / AR5523 */
        AR5523_DEVICE_UG(0x1385, 0x4250),        /* Netgear3 / WG111T (2) */
        AR5523_DEVICE_UG(0x1385, 0x5f00),        /* Netgear / WPN111 */
        AR5523_DEVICE_UG(0x1385, 0x5f02),        /* Netgear / WPN111 */
        { }
};
MODULE_DEVICE_TABLE(usb, ar5523_id_table);

static struct usb_driver ar5523_driver = {
        .name                = "ar5523",
        .id_table        = ar5523_id_table,
        .probe                = ar5523_probe,
        .disconnect        = ar5523_disconnect,
};

module_usb_driver(ar5523_driver);

MODULE_DESCRIPTION("Atheros AR5523 wireless driver");
MODULE_LICENSE("Dual BSD/GPL");
MODULE_FIRMWARE(AR5523_FIRMWARE_FILE);






























































































    4 




























    1 








    2 













































































































































































































    3 



























    3 










    1 


















    1 











   16 















    2 





    1 




    2 


    4 

    4 






   10 
    2 







    2 





    4 



    4 


    1 


    3 








    5 
    2 




    2 
    3 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
/*
 * Copyright (c) 2010-2011 Atheros Communications Inc.
 *
 * Permission to use, copy, modify, and/or distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include "htc.h"

static int htc_issue_send(struct htc_target *target, struct sk_buff* skb,
                          u16 len, u8 flags, u8 epid)

{
        struct htc_frame_hdr *hdr;
        struct htc_endpoint *endpoint = &target->endpoint[epid];
        int status;

        hdr = skb_push(skb, sizeof(struct htc_frame_hdr));
        hdr->endpoint_id = epid;
        hdr->flags = flags;
        hdr->payload_len = cpu_to_be16(len);
        memset(hdr->control, 0, sizeof(hdr->control));

        status = target->hif->send(target->hif_dev, endpoint->ul_pipeid, skb);

        return status;
}

static struct htc_endpoint *get_next_avail_ep(struct htc_endpoint *endpoint)
{
        enum htc_endpoint_id avail_epid;

        for (avail_epid = (ENDPOINT_MAX - 1); avail_epid > ENDPOINT0; avail_epid--)
                if (endpoint[avail_epid].service_id == 0)
                        return &endpoint[avail_epid];
        return NULL;
}

static u8 service_to_ulpipe(u16 service_id)
{
        switch (service_id) {
        case WMI_CONTROL_SVC:
                return 4;
        case WMI_BEACON_SVC:
        case WMI_CAB_SVC:
        case WMI_UAPSD_SVC:
        case WMI_MGMT_SVC:
        case WMI_DATA_VO_SVC:
        case WMI_DATA_VI_SVC:
        case WMI_DATA_BE_SVC:
        case WMI_DATA_BK_SVC:
                return 1;
        default:
                return 0;
        }
}

static u8 service_to_dlpipe(u16 service_id)
{
        switch (service_id) {
        case WMI_CONTROL_SVC:
                return 3;
        case WMI_BEACON_SVC:
        case WMI_CAB_SVC:
        case WMI_UAPSD_SVC:
        case WMI_MGMT_SVC:
        case WMI_DATA_VO_SVC:
        case WMI_DATA_VI_SVC:
        case WMI_DATA_BE_SVC:
        case WMI_DATA_BK_SVC:
                return 2;
        default:
                return 0;
        }
}

static void htc_process_target_rdy(struct htc_target *target,
                                   void *buf)
{
        struct htc_endpoint *endpoint;
        struct htc_ready_msg *htc_ready_msg = buf;

        target->credit_size = be16_to_cpu(htc_ready_msg->credit_size);

        endpoint = &target->endpoint[ENDPOINT0];
        endpoint->service_id = HTC_CTRL_RSVD_SVC;
        endpoint->max_msglen = HTC_MAX_CONTROL_MESSAGE_LENGTH;
        atomic_inc(&target->tgt_ready);
        complete(&target->target_wait);
}

static void htc_process_conn_rsp(struct htc_target *target,
                                 struct htc_frame_hdr *htc_hdr)
{
        struct htc_conn_svc_rspmsg *svc_rspmsg;
        struct htc_endpoint *endpoint, *tmp_endpoint = NULL;
        u16 service_id;
        u16 max_msglen;
        enum htc_endpoint_id epid, tepid;

        svc_rspmsg = (struct htc_conn_svc_rspmsg *)
                ((void *) htc_hdr + sizeof(struct htc_frame_hdr));

        if (svc_rspmsg->status == HTC_SERVICE_SUCCESS) {
                epid = svc_rspmsg->endpoint_id;

                /* Check that the received epid for the endpoint to attach
                 * a new service is valid. ENDPOINT0 can't be used here as it
                 * is already reserved for HTC_CTRL_RSVD_SVC service and thus
                 * should not be modified.
                 */
                if (epid <= ENDPOINT0 || epid >= ENDPOINT_MAX)
                        return;

                service_id = be16_to_cpu(svc_rspmsg->service_id);
                max_msglen = be16_to_cpu(svc_rspmsg->max_msg_len);
                endpoint = &target->endpoint[epid];

                for (tepid = (ENDPOINT_MAX - 1); tepid > ENDPOINT0; tepid--) {
                        tmp_endpoint = &target->endpoint[tepid];
                        if (tmp_endpoint->service_id == service_id) {
                                tmp_endpoint->service_id = 0;
                                break;
                        }
                }

                if (tepid == ENDPOINT0)
                        return;

                endpoint->service_id = service_id;
                endpoint->max_txqdepth = tmp_endpoint->max_txqdepth;
                endpoint->ep_callbacks = tmp_endpoint->ep_callbacks;
                endpoint->ul_pipeid = tmp_endpoint->ul_pipeid;
                endpoint->dl_pipeid = tmp_endpoint->dl_pipeid;
                endpoint->max_msglen = max_msglen;
                target->conn_rsp_epid = epid;
                complete(&target->cmd_wait);
        } else {
                target->conn_rsp_epid = ENDPOINT_UNUSED;
        }
}

static int htc_config_pipe_credits(struct htc_target *target)
{
        struct sk_buff *skb;
        struct htc_config_pipe_msg *cp_msg;
        int ret;
        unsigned long time_left;

        skb = alloc_skb(50 + sizeof(struct htc_frame_hdr), GFP_ATOMIC);
        if (!skb) {
                dev_err(target->dev, "failed to allocate send buffer\n");
                return -ENOMEM;
        }
        skb_reserve(skb, sizeof(struct htc_frame_hdr));

        cp_msg = skb_put(skb, sizeof(struct htc_config_pipe_msg));

        cp_msg->message_id = cpu_to_be16(HTC_MSG_CONFIG_PIPE_ID);
        cp_msg->pipe_id = USB_WLAN_TX_PIPE;
        cp_msg->credits = target->credits;

        target->htc_flags |= HTC_OP_CONFIG_PIPE_CREDITS;

        ret = htc_issue_send(target, skb, skb->len, 0, ENDPOINT0);
        if (ret)
                goto err;

        time_left = wait_for_completion_timeout(&target->cmd_wait, HZ);
        if (!time_left) {
                dev_err(target->dev, "HTC credit config timeout\n");
                return -ETIMEDOUT;
        }

        return 0;
err:
        kfree_skb(skb);
        return -EINVAL;
}

static int htc_setup_complete(struct htc_target *target)
{
        struct sk_buff *skb;
        struct htc_comp_msg *comp_msg;
        int ret = 0;
        unsigned long time_left;

        skb = alloc_skb(50 + sizeof(struct htc_frame_hdr), GFP_ATOMIC);
        if (!skb) {
                dev_err(target->dev, "failed to allocate send buffer\n");
                return -ENOMEM;
        }
        skb_reserve(skb, sizeof(struct htc_frame_hdr));

        comp_msg = skb_put(skb, sizeof(struct htc_comp_msg));
        comp_msg->msg_id = cpu_to_be16(HTC_MSG_SETUP_COMPLETE_ID);

        target->htc_flags |= HTC_OP_START_WAIT;

        ret = htc_issue_send(target, skb, skb->len, 0, ENDPOINT0);
        if (ret)
                goto err;

        time_left = wait_for_completion_timeout(&target->cmd_wait, HZ);
        if (!time_left) {
                dev_err(target->dev, "HTC start timeout\n");
                return -ETIMEDOUT;
        }

        return 0;

err:
        kfree_skb(skb);
        return -EINVAL;
}

/* HTC APIs */

int htc_init(struct htc_target *target)
{
        int ret;

        ret = htc_config_pipe_credits(target);
        if (ret)
                return ret;

        return htc_setup_complete(target);
}

int htc_connect_service(struct htc_target *target,
                     struct htc_service_connreq *service_connreq,
                     enum htc_endpoint_id *conn_rsp_epid)
{
        struct sk_buff *skb;
        struct htc_endpoint *endpoint;
        struct htc_conn_svc_msg *conn_msg;
        int ret;
        unsigned long time_left;

        /* Find an available endpoint */
        endpoint = get_next_avail_ep(target->endpoint);
        if (!endpoint) {
                dev_err(target->dev, "Endpoint is not available for service %d\n",
                        service_connreq->service_id);
                return -EINVAL;
        }

        endpoint->service_id = service_connreq->service_id;
        endpoint->max_txqdepth = service_connreq->max_send_qdepth;
        endpoint->ul_pipeid = service_to_ulpipe(service_connreq->service_id);
        endpoint->dl_pipeid = service_to_dlpipe(service_connreq->service_id);
        endpoint->ep_callbacks = service_connreq->ep_callbacks;

        skb = alloc_skb(sizeof(struct htc_conn_svc_msg) +
                            sizeof(struct htc_frame_hdr), GFP_ATOMIC);
        if (!skb) {
                dev_err(target->dev, "Failed to allocate buf to send"
                        "service connect req\n");
                return -ENOMEM;
        }

        skb_reserve(skb, sizeof(struct htc_frame_hdr));

        conn_msg = skb_put(skb, sizeof(struct htc_conn_svc_msg));
        conn_msg->service_id = cpu_to_be16(service_connreq->service_id);
        conn_msg->msg_id = cpu_to_be16(HTC_MSG_CONNECT_SERVICE_ID);
        conn_msg->con_flags = cpu_to_be16(service_connreq->con_flags);
        conn_msg->dl_pipeid = endpoint->dl_pipeid;
        conn_msg->ul_pipeid = endpoint->ul_pipeid;

        /* To prevent infoleak */
        conn_msg->svc_meta_len = 0;
        conn_msg->pad = 0;

        ret = htc_issue_send(target, skb, skb->len, 0, ENDPOINT0);
        if (ret)
                goto err;

        time_left = wait_for_completion_timeout(&target->cmd_wait, HZ);
        if (!time_left) {
                dev_err(target->dev, "Service connection timeout for: %d\n",
                        service_connreq->service_id);
                return -ETIMEDOUT;
        }

        if (target->conn_rsp_epid < 0 || target->conn_rsp_epid >= ENDPOINT_MAX)
                return -EINVAL;

        *conn_rsp_epid = target->conn_rsp_epid;
        return 0;
err:
        kfree_skb(skb);
        return ret;
}

int htc_send(struct htc_target *target, struct sk_buff *skb)
{
        struct ath9k_htc_tx_ctl *tx_ctl;

        tx_ctl = HTC_SKB_CB(skb);
        return htc_issue_send(target, skb, skb->len, 0, tx_ctl->epid);
}

int htc_send_epid(struct htc_target *target, struct sk_buff *skb,
                  enum htc_endpoint_id epid)
{
        return htc_issue_send(target, skb, skb->len, 0, epid);
}

void htc_stop(struct htc_target *target)
{
        target->hif->stop(target->hif_dev);
}

void htc_start(struct htc_target *target)
{
        target->hif->start(target->hif_dev);
}

void htc_sta_drain(struct htc_target *target, u8 idx)
{
        target->hif->sta_drain(target->hif_dev, idx);
}

void ath9k_htc_txcompletion_cb(struct htc_target *htc_handle,
                               struct sk_buff *skb, bool txok)
{
        struct htc_endpoint *endpoint;
        struct htc_frame_hdr *htc_hdr = NULL;

        if (htc_handle->htc_flags & HTC_OP_CONFIG_PIPE_CREDITS) {
                complete(&htc_handle->cmd_wait);
                htc_handle->htc_flags &= ~HTC_OP_CONFIG_PIPE_CREDITS;
                goto ret;
        }

        if (htc_handle->htc_flags & HTC_OP_START_WAIT) {
                complete(&htc_handle->cmd_wait);
                htc_handle->htc_flags &= ~HTC_OP_START_WAIT;
                goto ret;
        }

        if (skb) {
                htc_hdr = (struct htc_frame_hdr *) skb->data;
                if (htc_hdr->endpoint_id >= ARRAY_SIZE(htc_handle->endpoint))
                        goto ret;
                endpoint = &htc_handle->endpoint[htc_hdr->endpoint_id];
                skb_pull(skb, sizeof(struct htc_frame_hdr));

                if (endpoint->ep_callbacks.tx) {
                        endpoint->ep_callbacks.tx(endpoint->ep_callbacks.priv,
                                                  skb, htc_hdr->endpoint_id,
                                                  txok);
                } else {
                        kfree_skb(skb);
                }
        }

        return;
ret:
        kfree_skb(skb);
}

static void ath9k_htc_fw_panic_report(struct htc_target *htc_handle,
                                      struct sk_buff *skb, u32 len)
{
        uint32_t *pattern = (uint32_t *)skb->data;

        if (*pattern == 0x33221199 && len >= sizeof(struct htc_panic_bad_vaddr)) {
                struct htc_panic_bad_vaddr *htc_panic;
                htc_panic = (struct htc_panic_bad_vaddr *) skb->data;
                dev_err(htc_handle->dev, "ath: firmware panic! "
                        "exccause: 0x%08x; pc: 0x%08x; badvaddr: 0x%08x.\n",
                        htc_panic->exccause, htc_panic->pc,
                        htc_panic->badvaddr);
                return;
        }
        if (*pattern == 0x33221299) {
                struct htc_panic_bad_epid *htc_panic;
                htc_panic = (struct htc_panic_bad_epid *) skb->data;
                dev_err(htc_handle->dev, "ath: firmware panic! "
                        "bad epid: 0x%08x\n", htc_panic->epid);
                return;
        }
        dev_err(htc_handle->dev, "ath: unknown panic pattern!\n");
}

/*
 * HTC Messages are handled directly here and the obtained SKB
 * is freed.
 *
 * Service messages (Data, WMI) are passed to the corresponding
 * endpoint RX handlers, which have to free the SKB.
 */
void ath9k_htc_rx_msg(struct htc_target *htc_handle,
                      struct sk_buff *skb, u32 len, u8 pipe_id)
{
        struct htc_frame_hdr *htc_hdr;
        enum htc_endpoint_id epid;
        struct htc_endpoint *endpoint;
        __be16 *msg_id;

        if (!htc_handle || !skb)
                return;

        /* A valid message requires len >= 8.
         *
         *   sizeof(struct htc_frame_hdr) == 8
         *   sizeof(struct htc_ready_msg) == 8
         *   sizeof(struct htc_panic_bad_vaddr) == 16
         *   sizeof(struct htc_panic_bad_epid) == 8
         */
        if (unlikely(len < sizeof(struct htc_frame_hdr)))
                goto invalid;
        htc_hdr = (struct htc_frame_hdr *) skb->data;
        epid = htc_hdr->endpoint_id;

        if (epid == 0x99) {
                ath9k_htc_fw_panic_report(htc_handle, skb, len);
                kfree_skb(skb);
                return;
        }

        if (epid < 0 || epid >= ENDPOINT_MAX) {
invalid:
                if (pipe_id != USB_REG_IN_PIPE)
                        dev_kfree_skb_any(skb);
                else
                        kfree_skb(skb);
                return;
        }

        if (epid == ENDPOINT0) {

                /* Handle trailer */
                if (htc_hdr->flags & HTC_FLAGS_RECV_TRAILER) {
                        if (be32_to_cpu(*(__be32 *) skb->data) == 0x00C60000) {
                                /* Move past the Watchdog pattern */
                                htc_hdr = (struct htc_frame_hdr *)(skb->data + 4);
                                len -= 4;
                        }
                }

                /* Get the message ID */
                if (unlikely(len < sizeof(struct htc_frame_hdr) + sizeof(__be16)))
                        goto invalid;
                msg_id = (__be16 *) ((void *) htc_hdr +
                                     sizeof(struct htc_frame_hdr));

                /* Now process HTC messages */
                switch (be16_to_cpu(*msg_id)) {
                case HTC_MSG_READY_ID:
                        if (unlikely(len < sizeof(struct htc_ready_msg)))
                                goto invalid;
                        htc_process_target_rdy(htc_handle, htc_hdr);
                        break;
                case HTC_MSG_CONNECT_SERVICE_RESPONSE_ID:
                        if (unlikely(len < sizeof(struct htc_frame_hdr) +
                                     sizeof(struct htc_conn_svc_rspmsg)))
                                goto invalid;
                        htc_process_conn_rsp(htc_handle, htc_hdr);
                        break;
                default:
                        break;
                }

                kfree_skb(skb);

        } else {
                if (htc_hdr->flags & HTC_FLAGS_RECV_TRAILER)
                        skb_trim(skb, len - htc_hdr->control[0]);

                skb_pull(skb, sizeof(struct htc_frame_hdr));

                endpoint = &htc_handle->endpoint[epid];
                if (endpoint->ep_callbacks.rx)
                        endpoint->ep_callbacks.rx(endpoint->ep_callbacks.priv,
                                                  skb, epid);
                else
                        goto invalid;
        }
}

struct htc_target *ath9k_htc_hw_alloc(void *hif_handle,
                                      struct ath9k_htc_hif *hif,
                                      struct device *dev)
{
        struct htc_endpoint *endpoint;
        struct htc_target *target;

        target = kzalloc(sizeof(struct htc_target), GFP_KERNEL);
        if (!target)
                return NULL;

        init_completion(&target->target_wait);
        init_completion(&target->cmd_wait);

        target->hif = hif;
        target->hif_dev = hif_handle;
        target->dev = dev;

        /* Assign control endpoint pipe IDs */
        endpoint = &target->endpoint[ENDPOINT0];
        endpoint->ul_pipeid = hif->control_ul_pipe;
        endpoint->dl_pipeid = hif->control_dl_pipe;

        atomic_set(&target->tgt_ready, 0);

        return target;
}

void ath9k_htc_hw_free(struct htc_target *htc)
{
        kfree(htc);
}

int ath9k_htc_hw_init(struct htc_target *target,
                      struct device *dev, u16 devid,
                      char *product, u32 drv_info)
{
        if (ath9k_htc_probe_device(target, dev, devid, product, drv_info)) {
                pr_err("Failed to initialize the device\n");
                return -ENODEV;
        }

        return 0;
}

void ath9k_htc_hw_deinit(struct htc_target *target, bool hot_unplug)
{
        if (target)
                ath9k_htc_disconnect_device(target, hot_unplug);
}





















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PROFILE_H
#define _LINUX_PROFILE_H

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/cache.h>

#include <asm/errno.h>

#define CPU_PROFILING        1
#define SCHED_PROFILING        2
#define KVM_PROFILING        4

struct proc_dir_entry;
struct notifier_block;

#if defined(CONFIG_PROFILING) && defined(CONFIG_PROC_FS)
int create_proc_profile(void);
#else
static inline int create_proc_profile(void)
{
        return 0;
}
#endif

#ifdef CONFIG_PROFILING

extern int prof_on __read_mostly;

/* init basic kernel profiler */
int profile_init(void);
int profile_setup(char *str);
void profile_tick(int type);
int setup_profiling_timer(unsigned int multiplier);

/*
 * Add multiple profiler hits to a given address:
 */
void profile_hits(int type, void *ip, unsigned int nr_hits);

/*
 * Single profiler hit:
 */
static inline void profile_hit(int type, void *ip)
{
        /*
         * Speedup for the common (no profiling enabled) case:
         */
        if (unlikely(prof_on == type))
                profile_hits(type, ip, 1);
}

struct task_struct;
struct mm_struct;

#else

#define prof_on 0

static inline int profile_init(void)
{
        return 0;
}

static inline void profile_tick(int type)
{
        return;
}

static inline void profile_hits(int type, void *ip, unsigned int nr_hits)
{
        return;
}

static inline void profile_hit(int type, void *ip)
{
        return;
}


#endif /* CONFIG_PROFILING */

#endif /* _LINUX_PROFILE_H */
































































































    9 


   12 
















   22 



   16 



   13 



   21 




   22 
























   13 







   22 



   13 






   22 












   22 
















   22 








   13 








   13 














   22 
















   21 



























    2 













    2 










   21 






    3 


    9 



   13 


   21 






 2624 





 2618 

 2620 





  193 




  851 









   13 








    1 



    1 







    8 

    4 
















   14 




    1 









   13 




   13 










   14 




    1 



   13 


   13 






    6 

















    6 






















   10 


   10 






   10 




   11 






   11 






























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/sch_cbs.c        Credit Based Shaper
 *
 * Authors:        Vinicius Costa Gomes <vinicius.gomes@intel.com>
 */

/* Credit Based Shaper (CBS)
 * =========================
 *
 * This is a simple rate-limiting shaper aimed at TSN applications on
 * systems with known traffic workloads.
 *
 * Its algorithm is defined by the IEEE 802.1Q-2014 Specification,
 * Section 8.6.8.2, and explained in more detail in the Annex L of the
 * same specification.
 *
 * There are four tunables to be considered:
 *
 *        'idleslope': Idleslope is the rate of credits that is
 *        accumulated (in kilobits per second) when there is at least
 *        one packet waiting for transmission. Packets are transmitted
 *        when the current value of credits is equal or greater than
 *        zero. When there is no packet to be transmitted the amount of
 *        credits is set to zero. This is the main tunable of the CBS
 *        algorithm.
 *
 *        'sendslope':
 *        Sendslope is the rate of credits that is depleted (it should be a
 *        negative number of kilobits per second) when a transmission is
 *        ocurring. It can be calculated as follows, (IEEE 802.1Q-2014 Section
 *        8.6.8.2 item g):
 *
 *        sendslope = idleslope - port_transmit_rate
 *
 *        'hicredit': Hicredit defines the maximum amount of credits (in
 *        bytes) that can be accumulated. Hicredit depends on the
 *        characteristics of interfering traffic,
 *        'max_interference_size' is the maximum size of any burst of
 *        traffic that can delay the transmission of a frame that is
 *        available for transmission for this traffic class, (IEEE
 *        802.1Q-2014 Annex L, Equation L-3):
 *
 *        hicredit = max_interference_size * (idleslope / port_transmit_rate)
 *
 *        'locredit': Locredit is the minimum amount of credits that can
 *        be reached. It is a function of the traffic flowing through
 *        this qdisc (IEEE 802.1Q-2014 Annex L, Equation L-2):
 *
 *        locredit = max_frame_size * (sendslope / port_transmit_rate)
 */

#include <linux/ethtool.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/units.h>

#include <net/netevent.h>
#include <net/netlink.h>
#include <net/sch_generic.h>
#include <net/pkt_sched.h>

static LIST_HEAD(cbs_list);
static DEFINE_SPINLOCK(cbs_list_lock);

struct cbs_sched_data {
        bool offload;
        int queue;
        atomic64_t port_rate; /* in bytes/s */
        s64 last; /* timestamp in ns */
        s64 credits; /* in bytes */
        s32 locredit; /* in bytes */
        s32 hicredit; /* in bytes */
        s64 sendslope; /* in bytes/s */
        s64 idleslope; /* in bytes/s */
        struct qdisc_watchdog watchdog;
        int (*enqueue)(struct sk_buff *skb, struct Qdisc *sch,
                       struct sk_buff **to_free);
        struct sk_buff *(*dequeue)(struct Qdisc *sch);
        struct Qdisc *qdisc;
        struct list_head cbs_list;
};

static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch,
                             struct Qdisc *child,
                             struct sk_buff **to_free)
{
        unsigned int len = qdisc_pkt_len(skb);
        int err;

        err = child->ops->enqueue(skb, child, to_free);
        if (err != NET_XMIT_SUCCESS)
                return err;

        sch->qstats.backlog += len;
        sch->q.qlen++;

        return NET_XMIT_SUCCESS;
}

static int cbs_enqueue_offload(struct sk_buff *skb, struct Qdisc *sch,
                               struct sk_buff **to_free)
{
        struct cbs_sched_data *q = qdisc_priv(sch);
        struct Qdisc *qdisc = q->qdisc;

        return cbs_child_enqueue(skb, sch, qdisc, to_free);
}

static int cbs_enqueue_soft(struct sk_buff *skb, struct Qdisc *sch,
                            struct sk_buff **to_free)
{
        struct cbs_sched_data *q = qdisc_priv(sch);
        struct Qdisc *qdisc = q->qdisc;

        if (sch->q.qlen == 0 && q->credits > 0) {
                /* We need to stop accumulating credits when there's
                 * no enqueued packets and q->credits is positive.
                 */
                q->credits = 0;
                q->last = ktime_get_ns();
        }

        return cbs_child_enqueue(skb, sch, qdisc, to_free);
}

static int cbs_enqueue(struct sk_buff *skb, struct Qdisc *sch,
                       struct sk_buff **to_free)
{
        struct cbs_sched_data *q = qdisc_priv(sch);

        return q->enqueue(skb, sch, to_free);
}

/* timediff is in ns, slope is in bytes/s */
static s64 timediff_to_credits(s64 timediff, s64 slope)
{
        return div64_s64(timediff * slope, NSEC_PER_SEC);
}

static s64 delay_from_credits(s64 credits, s64 slope)
{
        if (unlikely(slope == 0))
                return S64_MAX;

        return div64_s64(-credits * NSEC_PER_SEC, slope);
}

static s64 credits_from_len(unsigned int len, s64 slope, s64 port_rate)
{
        if (unlikely(port_rate == 0))
                return S64_MAX;

        return div64_s64(len * slope, port_rate);
}

static struct sk_buff *cbs_child_dequeue(struct Qdisc *sch, struct Qdisc *child)
{
        struct sk_buff *skb;

        skb = child->ops->dequeue(child);
        if (!skb)
                return NULL;

        qdisc_qstats_backlog_dec(sch, skb);
        qdisc_bstats_update(sch, skb);
        sch->q.qlen--;

        return skb;
}

static struct sk_buff *cbs_dequeue_soft(struct Qdisc *sch)
{
        struct cbs_sched_data *q = qdisc_priv(sch);
        struct Qdisc *qdisc = q->qdisc;
        s64 now = ktime_get_ns();
        struct sk_buff *skb;
        s64 credits;
        int len;

        /* The previous packet is still being sent */
        if (now < q->last) {
                qdisc_watchdog_schedule_ns(&q->watchdog, q->last);
                return NULL;
        }
        if (q->credits < 0) {
                credits = timediff_to_credits(now - q->last, q->idleslope);

                credits = q->credits + credits;
                q->credits = min_t(s64, credits, q->hicredit);

                if (q->credits < 0) {
                        s64 delay;

                        delay = delay_from_credits(q->credits, q->idleslope);
                        qdisc_watchdog_schedule_ns(&q->watchdog, now + delay);

                        q->last = now;

                        return NULL;
                }
        }
        skb = cbs_child_dequeue(sch, qdisc);
        if (!skb)
                return NULL;

        len = qdisc_pkt_len(skb);

        /* As sendslope is a negative number, this will decrease the
         * amount of q->credits.
         */
        credits = credits_from_len(len, q->sendslope,
                                   atomic64_read(&q->port_rate));
        credits += q->credits;

        q->credits = max_t(s64, credits, q->locredit);
        /* Estimate of the transmission of the last byte of the packet in ns */
        if (unlikely(atomic64_read(&q->port_rate) == 0))
                q->last = now;
        else
                q->last = now + div64_s64(len * NSEC_PER_SEC,
                                          atomic64_read(&q->port_rate));

        return skb;
}

static struct sk_buff *cbs_dequeue_offload(struct Qdisc *sch)
{
        struct cbs_sched_data *q = qdisc_priv(sch);
        struct Qdisc *qdisc = q->qdisc;

        return cbs_child_dequeue(sch, qdisc);
}

static struct sk_buff *cbs_dequeue(struct Qdisc *sch)
{
        struct cbs_sched_data *q = qdisc_priv(sch);

        return q->dequeue(sch);
}

static const struct nla_policy cbs_policy[TCA_CBS_MAX + 1] = {
        [TCA_CBS_PARMS]        = { .len = sizeof(struct tc_cbs_qopt) },
};

static void cbs_disable_offload(struct net_device *dev,
                                struct cbs_sched_data *q)
{
        struct tc_cbs_qopt_offload cbs = { };
        const struct net_device_ops *ops;
        int err;

        if (!q->offload)
                return;

        q->enqueue = cbs_enqueue_soft;
        q->dequeue = cbs_dequeue_soft;

        ops = dev->netdev_ops;
        if (!ops->ndo_setup_tc)
                return;

        cbs.queue = q->queue;
        cbs.enable = 0;

        err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_CBS, &cbs);
        if (err < 0)
                pr_warn("Couldn't disable CBS offload for queue %d\n",
                        cbs.queue);
}

static int cbs_enable_offload(struct net_device *dev, struct cbs_sched_data *q,
                              const struct tc_cbs_qopt *opt,
                              struct netlink_ext_ack *extack)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        struct tc_cbs_qopt_offload cbs = { };
        int err;

        if (!ops->ndo_setup_tc) {
                NL_SET_ERR_MSG(extack, "Specified device does not support cbs offload");
                return -EOPNOTSUPP;
        }

        cbs.queue = q->queue;

        cbs.enable = 1;
        cbs.hicredit = opt->hicredit;
        cbs.locredit = opt->locredit;
        cbs.idleslope = opt->idleslope;
        cbs.sendslope = opt->sendslope;

        err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_CBS, &cbs);
        if (err < 0) {
                NL_SET_ERR_MSG(extack, "Specified device failed to setup cbs hardware offload");
                return err;
        }

        q->enqueue = cbs_enqueue_offload;
        q->dequeue = cbs_dequeue_offload;

        return 0;
}

static void cbs_set_port_rate(struct net_device *dev, struct cbs_sched_data *q)
{
        struct ethtool_link_ksettings ecmd;
        int speed = SPEED_10;
        s64 port_rate;
        int err;

        err = __ethtool_get_link_ksettings(dev, &ecmd);
        if (err < 0)
                goto skip;

        if (ecmd.base.speed && ecmd.base.speed != SPEED_UNKNOWN)
                speed = ecmd.base.speed;

skip:
        port_rate = speed * 1000 * BYTES_PER_KBIT;

        atomic64_set(&q->port_rate, port_rate);
        netdev_dbg(dev, "cbs: set %s's port_rate to: %lld, linkspeed: %d\n",
                   dev->name, (long long)atomic64_read(&q->port_rate),
                   ecmd.base.speed);
}

static int cbs_dev_notifier(struct notifier_block *nb, unsigned long event,
                            void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct cbs_sched_data *q;
        struct net_device *qdev;
        bool found = false;

        ASSERT_RTNL();

        if (event != NETDEV_UP && event != NETDEV_CHANGE)
                return NOTIFY_DONE;

        spin_lock(&cbs_list_lock);
        list_for_each_entry(q, &cbs_list, cbs_list) {
                qdev = qdisc_dev(q->qdisc);
                if (qdev == dev) {
                        found = true;
                        break;
                }
        }
        spin_unlock(&cbs_list_lock);

        if (found)
                cbs_set_port_rate(dev, q);

        return NOTIFY_DONE;
}

static int cbs_change(struct Qdisc *sch, struct nlattr *opt,
                      struct netlink_ext_ack *extack)
{
        struct cbs_sched_data *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct nlattr *tb[TCA_CBS_MAX + 1];
        struct tc_cbs_qopt *qopt;
        int err;

        err = nla_parse_nested_deprecated(tb, TCA_CBS_MAX, opt, cbs_policy,
                                          extack);
        if (err < 0)
                return err;

        if (!tb[TCA_CBS_PARMS]) {
                NL_SET_ERR_MSG(extack, "Missing CBS parameter which are mandatory");
                return -EINVAL;
        }

        qopt = nla_data(tb[TCA_CBS_PARMS]);

        if (!qopt->offload) {
                cbs_set_port_rate(dev, q);
                cbs_disable_offload(dev, q);
        } else {
                err = cbs_enable_offload(dev, q, qopt, extack);
                if (err < 0)
                        return err;
        }

        /* Everything went OK, save the parameters used. */
        WRITE_ONCE(q->hicredit, qopt->hicredit);
        WRITE_ONCE(q->locredit, qopt->locredit);
        WRITE_ONCE(q->idleslope, qopt->idleslope * BYTES_PER_KBIT);
        WRITE_ONCE(q->sendslope, qopt->sendslope * BYTES_PER_KBIT);
        WRITE_ONCE(q->offload, qopt->offload);

        return 0;
}

static int cbs_init(struct Qdisc *sch, struct nlattr *opt,
                    struct netlink_ext_ack *extack)
{
        struct cbs_sched_data *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);

        if (!opt) {
                NL_SET_ERR_MSG(extack, "Missing CBS qdisc options  which are mandatory");
                return -EINVAL;
        }

        q->qdisc = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
                                     sch->handle, extack);
        if (!q->qdisc)
                return -ENOMEM;

        spin_lock(&cbs_list_lock);
        list_add(&q->cbs_list, &cbs_list);
        spin_unlock(&cbs_list_lock);

        qdisc_hash_add(q->qdisc, false);

        q->queue = sch->dev_queue - netdev_get_tx_queue(dev, 0);

        q->enqueue = cbs_enqueue_soft;
        q->dequeue = cbs_dequeue_soft;

        qdisc_watchdog_init(&q->watchdog, sch);

        return cbs_change(sch, opt, extack);
}

static void cbs_destroy(struct Qdisc *sch)
{
        struct cbs_sched_data *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);

        /* Nothing to do if we couldn't create the underlying qdisc */
        if (!q->qdisc)
                return;

        qdisc_watchdog_cancel(&q->watchdog);
        cbs_disable_offload(dev, q);

        spin_lock(&cbs_list_lock);
        list_del(&q->cbs_list);
        spin_unlock(&cbs_list_lock);

        qdisc_put(q->qdisc);
}

static int cbs_dump(struct Qdisc *sch, struct sk_buff *skb)
{
        struct cbs_sched_data *q = qdisc_priv(sch);
        struct tc_cbs_qopt opt = { };
        struct nlattr *nest;

        nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
        if (!nest)
                goto nla_put_failure;

        opt.hicredit = READ_ONCE(q->hicredit);
        opt.locredit = READ_ONCE(q->locredit);
        opt.sendslope = div64_s64(READ_ONCE(q->sendslope), BYTES_PER_KBIT);
        opt.idleslope = div64_s64(READ_ONCE(q->idleslope), BYTES_PER_KBIT);
        opt.offload = READ_ONCE(q->offload);

        if (nla_put(skb, TCA_CBS_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        return nla_nest_end(skb, nest);

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -1;
}

static int cbs_dump_class(struct Qdisc *sch, unsigned long cl,
                          struct sk_buff *skb, struct tcmsg *tcm)
{
        struct cbs_sched_data *q = qdisc_priv(sch);

        if (cl != 1 || !q->qdisc)        /* only one class */
                return -ENOENT;

        tcm->tcm_handle |= TC_H_MIN(1);
        tcm->tcm_info = q->qdisc->handle;

        return 0;
}

static int cbs_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
                     struct Qdisc **old, struct netlink_ext_ack *extack)
{
        struct cbs_sched_data *q = qdisc_priv(sch);

        if (!new) {
                new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
                                        sch->handle, NULL);
                if (!new)
                        new = &noop_qdisc;
        }

        *old = qdisc_replace(sch, new, &q->qdisc);
        return 0;
}

static struct Qdisc *cbs_leaf(struct Qdisc *sch, unsigned long arg)
{
        struct cbs_sched_data *q = qdisc_priv(sch);

        return q->qdisc;
}

static unsigned long cbs_find(struct Qdisc *sch, u32 classid)
{
        return 1;
}

static void cbs_walk(struct Qdisc *sch, struct qdisc_walker *walker)
{
        if (!walker->stop) {
                tc_qdisc_stats_dump(sch, 1, walker);
        }
}

static const struct Qdisc_class_ops cbs_class_ops = {
        .graft                =        cbs_graft,
        .leaf                =        cbs_leaf,
        .find                =        cbs_find,
        .walk                =        cbs_walk,
        .dump                =        cbs_dump_class,
};

static struct Qdisc_ops cbs_qdisc_ops __read_mostly = {
        .id                =        "cbs",
        .cl_ops                =        &cbs_class_ops,
        .priv_size        =        sizeof(struct cbs_sched_data),
        .enqueue        =        cbs_enqueue,
        .dequeue        =        cbs_dequeue,
        .peek                =        qdisc_peek_dequeued,
        .init                =        cbs_init,
        .reset                =        qdisc_reset_queue,
        .destroy        =        cbs_destroy,
        .change                =        cbs_change,
        .dump                =        cbs_dump,
        .owner                =        THIS_MODULE,
};
MODULE_ALIAS_NET_SCH("cbs");

static struct notifier_block cbs_device_notifier = {
        .notifier_call = cbs_dev_notifier,
};

static int __init cbs_module_init(void)
{
        int err;

        err = register_netdevice_notifier(&cbs_device_notifier);
        if (err)
                return err;

        err = register_qdisc(&cbs_qdisc_ops);
        if (err)
                unregister_netdevice_notifier(&cbs_device_notifier);

        return err;
}

static void __exit cbs_module_exit(void)
{
        unregister_qdisc(&cbs_qdisc_ops);
        unregister_netdevice_notifier(&cbs_device_notifier);
}
module_init(cbs_module_init)
module_exit(cbs_module_exit)
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Credit Based shaper");









































































































    8 
   41 
   20 









   20 


   20 
   15 
   20 













   20 
    9 






















   67 

















    1 





   33 











   28 











   60 
    3 













   58 

   15 




   46 






   60 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
/*
 * Copyright (c) 2006 Oracle.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/percpu.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/proc_fs.h>
#include <linux/export.h>

#include "rds.h"

/*
 * This file implements a getsockopt() call which copies a set of fixed
 * sized structs into a user-specified buffer as a means of providing
 * read-only information about RDS.
 *
 * For a given information source there are a given number of fixed sized
 * structs at a given time.  The structs are only copied if the user-specified
 * buffer is big enough.  The destination pages that make up the buffer
 * are pinned for the duration of the copy.
 *
 * This gives us the following benefits:
 *
 * - simple implementation, no copy "position" across multiple calls
 * - consistent snapshot of an info source
 * - atomic copy works well with whatever locking info source has
 * - one portable tool to get rds info across implementations
 * - long-lived tool can get info without allocating
 *
 * at the following costs:
 *
 * - info source copy must be pinned, may be "large"
 */

struct rds_info_iterator {
        struct page **pages;
        void *addr;
        unsigned long offset;
};

static DEFINE_SPINLOCK(rds_info_lock);
static rds_info_func rds_info_funcs[RDS_INFO_LAST - RDS_INFO_FIRST + 1];

void rds_info_register_func(int optname, rds_info_func func)
{
        int offset = optname - RDS_INFO_FIRST;

        BUG_ON(optname < RDS_INFO_FIRST || optname > RDS_INFO_LAST);

        spin_lock(&rds_info_lock);
        BUG_ON(rds_info_funcs[offset]);
        rds_info_funcs[offset] = func;
        spin_unlock(&rds_info_lock);
}
EXPORT_SYMBOL_GPL(rds_info_register_func);

void rds_info_deregister_func(int optname, rds_info_func func)
{
        int offset = optname - RDS_INFO_FIRST;

        BUG_ON(optname < RDS_INFO_FIRST || optname > RDS_INFO_LAST);

        spin_lock(&rds_info_lock);
        BUG_ON(rds_info_funcs[offset] != func);
        rds_info_funcs[offset] = NULL;
        spin_unlock(&rds_info_lock);
}
EXPORT_SYMBOL_GPL(rds_info_deregister_func);

/*
 * Typically we hold an atomic kmap across multiple rds_info_copy() calls
 * because the kmap is so expensive.  This must be called before using blocking
 * operations while holding the mapping and as the iterator is torn down.
 */
void rds_info_iter_unmap(struct rds_info_iterator *iter)
{
        if (iter->addr) {
                kunmap_atomic(iter->addr);
                iter->addr = NULL;
        }
}

/*
 * get_user_pages() called flush_dcache_page() on the pages for us.
 */
void rds_info_copy(struct rds_info_iterator *iter, void *data,
                   unsigned long bytes)
{
        unsigned long this;

        while (bytes) {
                if (!iter->addr)
                        iter->addr = kmap_atomic(*iter->pages);

                this = min(bytes, PAGE_SIZE - iter->offset);

                rdsdebug("page %p addr %p offset %lu this %lu data %p "
                          "bytes %lu\n", *iter->pages, iter->addr,
                          iter->offset, this, data, bytes);

                memcpy(iter->addr + iter->offset, data, this);

                data += this;
                bytes -= this;
                iter->offset += this;

                if (iter->offset == PAGE_SIZE) {
                        kunmap_atomic(iter->addr);
                        iter->addr = NULL;
                        iter->offset = 0;
                        iter->pages++;
                }
        }
}
EXPORT_SYMBOL_GPL(rds_info_copy);

/*
 * @optval points to the userspace buffer that the information snapshot
 * will be copied into.
 *
 * @optlen on input is the size of the buffer in userspace.  @optlen
 * on output is the size of the requested snapshot in bytes.
 *
 * This function returns -errno if there is a failure, particularly -ENOSPC
 * if the given userspace buffer was not large enough to fit the snapshot.
 * On success it returns the positive number of bytes of each array element
 * in the snapshot.
 */
int rds_info_getsockopt(struct socket *sock, int optname, char __user *optval,
                        int __user *optlen)
{
        struct rds_info_iterator iter;
        struct rds_info_lengths lens;
        unsigned long nr_pages = 0;
        unsigned long start;
        rds_info_func func;
        struct page **pages = NULL;
        int ret;
        int len;
        int total;

        if (get_user(len, optlen)) {
                ret = -EFAULT;
                goto out;
        }

        /* check for all kinds of wrapping and the like */
        start = (unsigned long)optval;
        if (len < 0 || len > INT_MAX - PAGE_SIZE + 1 || start + len < start) {
                ret = -EINVAL;
                goto out;
        }

        /* a 0 len call is just trying to probe its length */
        if (len == 0)
                goto call_func;

        nr_pages = (PAGE_ALIGN(start + len) - (start & PAGE_MASK))
                        >> PAGE_SHIFT;

        pages = kmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
        if (!pages) {
                ret = -ENOMEM;
                goto out;
        }
        ret = pin_user_pages_fast(start, nr_pages, FOLL_WRITE, pages);
        if (ret != nr_pages) {
                if (ret > 0)
                        nr_pages = ret;
                else
                        nr_pages = 0;
                ret = -EAGAIN; /* XXX ? */
                goto out;
        }

        rdsdebug("len %d nr_pages %lu\n", len, nr_pages);

call_func:
        func = rds_info_funcs[optname - RDS_INFO_FIRST];
        if (!func) {
                ret = -ENOPROTOOPT;
                goto out;
        }

        iter.pages = pages;
        iter.addr = NULL;
        iter.offset = start & (PAGE_SIZE - 1);

        func(sock, len, &iter, &lens);
        BUG_ON(lens.each == 0);

        total = lens.nr * lens.each;

        rds_info_iter_unmap(&iter);

        if (total > len) {
                len = total;
                ret = -ENOSPC;
        } else {
                len = total;
                ret = lens.each;
        }

        if (put_user(len, optlen))
                ret = -EFAULT;

out:
        if (pages)
                unpin_user_pages(pages, nr_pages);
        kfree(pages);

        return ret;
}


























































































































































































    9 











  212 











  212 
































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
/*
 * Copyright (C) 2011-2013 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#ifndef DRM_RECT_H
#define DRM_RECT_H

#include <linux/types.h>

/**
 * DOC: rect utils
 *
 * Utility functions to help manage rectangular areas for
 * clipping, scaling, etc. calculations.
 */

/**
 * struct drm_rect - two dimensional rectangle
 * @x1: horizontal starting coordinate (inclusive)
 * @x2: horizontal ending coordinate (exclusive)
 * @y1: vertical starting coordinate (inclusive)
 * @y2: vertical ending coordinate (exclusive)
 *
 * Note that this must match the layout of struct drm_mode_rect or the damage
 * helpers like drm_atomic_helper_damage_iter_init() break.
 */
struct drm_rect {
        int x1, y1, x2, y2;
};

/**
 * DRM_RECT_INIT - initialize a rectangle from x/y/w/h
 * @x: x coordinate
 * @y: y coordinate
 * @w: width
 * @h: height
 *
 * RETURNS:
 * A new rectangle of the specified size.
 */
#define DRM_RECT_INIT(x, y, w, h) ((struct drm_rect){ \
                .x1 = (x), \
                .y1 = (y), \
                .x2 = (x) + (w), \
                .y2 = (y) + (h) })

/**
 * DRM_RECT_FMT - printf string for &struct drm_rect
 */
#define DRM_RECT_FMT    "%dx%d%+d%+d"
/**
 * DRM_RECT_ARG - printf arguments for &struct drm_rect
 * @r: rectangle struct
 */
#define DRM_RECT_ARG(r) drm_rect_width(r), drm_rect_height(r), (r)->x1, (r)->y1

/**
 * DRM_RECT_FP_FMT - printf string for &struct drm_rect in 16.16 fixed point
 */
#define DRM_RECT_FP_FMT "%d.%06ux%d.%06u%+d.%06u%+d.%06u"
/**
 * DRM_RECT_FP_ARG - printf arguments for &struct drm_rect in 16.16 fixed point
 * @r: rectangle struct
 *
 * This is useful for e.g. printing plane source rectangles, which are in 16.16
 * fixed point.
 */
#define DRM_RECT_FP_ARG(r) \
                drm_rect_width(r) >> 16, ((drm_rect_width(r) & 0xffff) * 15625) >> 10, \
                drm_rect_height(r) >> 16, ((drm_rect_height(r) & 0xffff) * 15625) >> 10, \
                (r)->x1 >> 16, (((r)->x1 & 0xffff) * 15625) >> 10, \
                (r)->y1 >> 16, (((r)->y1 & 0xffff) * 15625) >> 10

/**
 * drm_rect_init - initialize the rectangle from x/y/w/h
 * @r: rectangle
 * @x: x coordinate
 * @y: y coordinate
 * @width: width
 * @height: height
 */
static inline void drm_rect_init(struct drm_rect *r, int x, int y,
                                 int width, int height)
{
        r->x1 = x;
        r->y1 = y;
        r->x2 = x + width;
        r->y2 = y + height;
}

/**
 * drm_rect_adjust_size - adjust the size of the rectangle
 * @r: rectangle to be adjusted
 * @dw: horizontal adjustment
 * @dh: vertical adjustment
 *
 * Change the size of rectangle @r by @dw in the horizontal direction,
 * and by @dh in the vertical direction, while keeping the center
 * of @r stationary.
 *
 * Positive @dw and @dh increase the size, negative values decrease it.
 */
static inline void drm_rect_adjust_size(struct drm_rect *r, int dw, int dh)
{
        r->x1 -= dw >> 1;
        r->y1 -= dh >> 1;
        r->x2 += (dw + 1) >> 1;
        r->y2 += (dh + 1) >> 1;
}

/**
 * drm_rect_translate - translate the rectangle
 * @r: rectangle to be translated
 * @dx: horizontal translation
 * @dy: vertical translation
 *
 * Move rectangle @r by @dx in the horizontal direction,
 * and by @dy in the vertical direction.
 */
static inline void drm_rect_translate(struct drm_rect *r, int dx, int dy)
{
        r->x1 += dx;
        r->y1 += dy;
        r->x2 += dx;
        r->y2 += dy;
}

/**
 * drm_rect_translate_to - translate the rectangle to an absolute position
 * @r: rectangle to be translated
 * @x: horizontal position
 * @y: vertical position
 *
 * Move rectangle @r to @x in the horizontal direction,
 * and to @y in the vertical direction.
 */
static inline void drm_rect_translate_to(struct drm_rect *r, int x, int y)
{
        drm_rect_translate(r, x - r->x1, y - r->y1);
}

/**
 * drm_rect_downscale - downscale a rectangle
 * @r: rectangle to be downscaled
 * @horz: horizontal downscale factor
 * @vert: vertical downscale factor
 *
 * Divide the coordinates of rectangle @r by @horz and @vert.
 */
static inline void drm_rect_downscale(struct drm_rect *r, int horz, int vert)
{
        r->x1 /= horz;
        r->y1 /= vert;
        r->x2 /= horz;
        r->y2 /= vert;
}

/**
 * drm_rect_width - determine the rectangle width
 * @r: rectangle whose width is returned
 *
 * RETURNS:
 * The width of the rectangle.
 */
static inline int drm_rect_width(const struct drm_rect *r)
{
        return r->x2 - r->x1;
}

/**
 * drm_rect_height - determine the rectangle height
 * @r: rectangle whose height is returned
 *
 * RETURNS:
 * The height of the rectangle.
 */
static inline int drm_rect_height(const struct drm_rect *r)
{
        return r->y2 - r->y1;
}

/**
 * drm_rect_visible - determine if the rectangle is visible
 * @r: rectangle whose visibility is returned
 *
 * RETURNS:
 * %true if the rectangle is visible, %false otherwise.
 */
static inline bool drm_rect_visible(const struct drm_rect *r)
{
        return drm_rect_width(r) > 0 && drm_rect_height(r) > 0;
}

/**
 * drm_rect_equals - determine if two rectangles are equal
 * @r1: first rectangle
 * @r2: second rectangle
 *
 * RETURNS:
 * %true if the rectangles are equal, %false otherwise.
 */
static inline bool drm_rect_equals(const struct drm_rect *r1,
                                   const struct drm_rect *r2)
{
        return r1->x1 == r2->x1 && r1->x2 == r2->x2 &&
                r1->y1 == r2->y1 && r1->y2 == r2->y2;
}

/**
 * drm_rect_fp_to_int - Convert a rect in 16.16 fixed point form to int form.
 * @dst: rect to be stored the converted value
 * @src: rect in 16.16 fixed point form
 */
static inline void drm_rect_fp_to_int(struct drm_rect *dst,
                                      const struct drm_rect *src)
{
        drm_rect_init(dst, src->x1 >> 16, src->y1 >> 16,
                      drm_rect_width(src) >> 16,
                      drm_rect_height(src) >> 16);
}

/**
 * drm_rect_overlap - Check if two rectangles overlap
 * @a: first rectangle
 * @b: second rectangle
 *
 * RETURNS:
 * %true if the rectangles overlap, %false otherwise.
 */
static inline bool drm_rect_overlap(const struct drm_rect *a,
                                    const struct drm_rect *b)
{
        return (a->x2 > b->x1 && b->x2 > a->x1 &&
                a->y2 > b->y1 && b->y2 > a->y1);
}

bool drm_rect_intersect(struct drm_rect *r, const struct drm_rect *clip);
bool drm_rect_clip_scaled(struct drm_rect *src, struct drm_rect *dst,
                          const struct drm_rect *clip);
int drm_rect_calc_hscale(const struct drm_rect *src,
                         const struct drm_rect *dst,
                         int min_hscale, int max_hscale);
int drm_rect_calc_vscale(const struct drm_rect *src,
                         const struct drm_rect *dst,
                         int min_vscale, int max_vscale);
void drm_rect_debug_print(const char *prefix,
                          const struct drm_rect *r, bool fixed_point);
void drm_rect_rotate(struct drm_rect *r,
                     int width, int height,
                     unsigned int rotation);
void drm_rect_rotate_inv(struct drm_rect *r,
                         int width, int height,
                         unsigned int rotation);

#endif




































































































































  506 
























   10 


   10 













































   33 






   33 






































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
// SPDX-License-Identifier: GPL-2.0
/*
 * Creating audit events from TTY input.
 *
 * Copyright (C) 2007 Red Hat, Inc.  All rights reserved.
 *
 * Authors: Miloslav Trmac <mitr@redhat.com>
 */

#include <linux/audit.h>
#include <linux/slab.h>
#include <linux/tty.h>
#include "tty.h"

#define TTY_AUDIT_BUF_SIZE        4096

struct tty_audit_buf {
        struct mutex mutex;        /* Protects all data below */
        dev_t dev;                /* The TTY which the data is from */
        bool icanon;
        size_t valid;
        u8 *data;                /* Allocated size TTY_AUDIT_BUF_SIZE */
};

static struct tty_audit_buf *tty_audit_buf_ref(void)
{
        struct tty_audit_buf *buf;

        buf = current->signal->tty_audit_buf;
        WARN_ON(buf == ERR_PTR(-ESRCH));
        return buf;
}

static struct tty_audit_buf *tty_audit_buf_alloc(void)
{
        struct tty_audit_buf *buf;

        buf = kzalloc(sizeof(*buf), GFP_KERNEL);
        if (!buf)
                goto err;

        buf->data = kmalloc(TTY_AUDIT_BUF_SIZE, GFP_KERNEL);
        if (!buf->data)
                goto err_buf;

        mutex_init(&buf->mutex);

        return buf;

err_buf:
        kfree(buf);
err:
        return NULL;
}

static void tty_audit_buf_free(struct tty_audit_buf *buf)
{
        WARN_ON(buf->valid != 0);
        kfree(buf->data);
        kfree(buf);
}

static void tty_audit_log(const char *description, dev_t dev,
                          const u8 *data, size_t size)
{
        struct audit_buffer *ab;
        pid_t pid = task_pid_nr(current);
        uid_t uid = from_kuid(&init_user_ns, task_uid(current));
        uid_t loginuid = from_kuid(&init_user_ns, audit_get_loginuid(current));
        unsigned int sessionid = audit_get_sessionid(current);
        char name[TASK_COMM_LEN];

        ab = audit_log_start(audit_context(), GFP_KERNEL, AUDIT_TTY);
        if (!ab)
                return;

        audit_log_format(ab, "%s pid=%u uid=%u auid=%u ses=%u major=%d minor=%d comm=",
                         description, pid, uid, loginuid, sessionid,
                         MAJOR(dev), MINOR(dev));
        get_task_comm(name, current);
        audit_log_untrustedstring(ab, name);
        audit_log_format(ab, " data=");
        audit_log_n_hex(ab, data, size);
        audit_log_end(ab);
}

/*
 *        tty_audit_buf_push        -        Push buffered data out
 *
 *        Generate an audit message from the contents of @buf, which is owned by
 *        the current task.  @buf->mutex must be locked.
 */
static void tty_audit_buf_push(struct tty_audit_buf *buf)
{
        if (buf->valid == 0)
                return;
        if (audit_enabled == AUDIT_OFF) {
                buf->valid = 0;
                return;
        }
        tty_audit_log("tty", buf->dev, buf->data, buf->valid);
        buf->valid = 0;
}

/**
 *        tty_audit_exit        -        Handle a task exit
 *
 *        Make sure all buffered data is written out and deallocate the buffer.
 *        Only needs to be called if current->signal->tty_audit_buf != %NULL.
 *
 *        The process is single-threaded at this point; no other threads share
 *        current->signal.
 */
void tty_audit_exit(void)
{
        struct tty_audit_buf *buf;

        buf = xchg(&current->signal->tty_audit_buf, ERR_PTR(-ESRCH));
        if (!buf)
                return;

        tty_audit_buf_push(buf);
        tty_audit_buf_free(buf);
}

/*
 *        tty_audit_fork        -        Copy TTY audit state for a new task
 *
 *        Set up TTY audit state in @sig from current.  @sig needs no locking.
 */
void tty_audit_fork(struct signal_struct *sig)
{
        sig->audit_tty = current->signal->audit_tty;
}

/*
 *        tty_audit_tiocsti        -        Log TIOCSTI
 */
void tty_audit_tiocsti(const struct tty_struct *tty, u8 ch)
{
        dev_t dev;

        dev = MKDEV(tty->driver->major, tty->driver->minor_start) + tty->index;
        if (tty_audit_push())
                return;

        if (audit_enabled)
                tty_audit_log("ioctl=TIOCSTI", dev, &ch, 1);
}

/*
 *        tty_audit_push        -        Flush current's pending audit data
 *
 *        Returns 0 if success, -EPERM if tty audit is disabled
 */
int tty_audit_push(void)
{
        struct tty_audit_buf *buf;

        if (~current->signal->audit_tty & AUDIT_TTY_ENABLE)
                return -EPERM;

        buf = tty_audit_buf_ref();
        if (!IS_ERR_OR_NULL(buf)) {
                mutex_lock(&buf->mutex);
                tty_audit_buf_push(buf);
                mutex_unlock(&buf->mutex);
        }
        return 0;
}

/*
 *        tty_audit_buf_get        -        Get an audit buffer.
 *
 *        Get an audit buffer, allocate it if necessary.  Return %NULL
 *        if out of memory or ERR_PTR(-ESRCH) if tty_audit_exit() has already
 *        occurred.  Otherwise, return a new reference to the buffer.
 */
static struct tty_audit_buf *tty_audit_buf_get(void)
{
        struct tty_audit_buf *buf;

        buf = tty_audit_buf_ref();
        if (buf)
                return buf;

        buf = tty_audit_buf_alloc();
        if (buf == NULL) {
                audit_log_lost("out of memory in TTY auditing");
                return NULL;
        }

        /* Race to use this buffer, free it if another wins */
        if (cmpxchg(&current->signal->tty_audit_buf, NULL, buf) != NULL)
                tty_audit_buf_free(buf);
        return tty_audit_buf_ref();
}

/*
 *        tty_audit_add_data        -        Add data for TTY auditing.
 *
 *        Audit @data of @size from @tty, if necessary.
 */
void tty_audit_add_data(const struct tty_struct *tty, const void *data,
                        size_t size)
{
        struct tty_audit_buf *buf;
        unsigned int audit_tty;
        bool icanon = L_ICANON(tty);
        dev_t dev;

        audit_tty = READ_ONCE(current->signal->audit_tty);
        if (~audit_tty & AUDIT_TTY_ENABLE)
                return;

        if (unlikely(size == 0))
                return;

        if (tty->driver->type == TTY_DRIVER_TYPE_PTY
            && tty->driver->subtype == PTY_TYPE_MASTER)
                return;

        if ((~audit_tty & AUDIT_TTY_LOG_PASSWD) && icanon && !L_ECHO(tty))
                return;

        buf = tty_audit_buf_get();
        if (IS_ERR_OR_NULL(buf))
                return;

        mutex_lock(&buf->mutex);
        dev = MKDEV(tty->driver->major, tty->driver->minor_start) + tty->index;
        if (buf->dev != dev || buf->icanon != icanon) {
                tty_audit_buf_push(buf);
                buf->dev = dev;
                buf->icanon = icanon;
        }
        do {
                size_t run;

                run = TTY_AUDIT_BUF_SIZE - buf->valid;
                if (run > size)
                        run = size;
                memcpy(buf->data + buf->valid, data, run);
                buf->valid += run;
                data += run;
                size -= run;
                if (buf->valid == TTY_AUDIT_BUF_SIZE)
                        tty_audit_buf_push(buf);
        } while (size != 0);
        mutex_unlock(&buf->mutex);
}

































































































































































    1 






    1 






    1 



































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
// SPDX-License-Identifier: GPL-2.0
/*
 * Renesas Electronics uPD78F0730 USB to serial converter driver
 *
 * Copyright (C) 2014,2016 Maksim Salau <maksim.salau@gmail.com>
 *
 * Protocol of the adaptor is described in the application note U19660EJ1V0AN00
 * μPD78F0730 8-bit Single-Chip Microcontroller
 * USB-to-Serial Conversion Software
 * <https://www.renesas.com/en-eu/doc/DocumentServer/026/U19660EJ1V0AN00.pdf>
 *
 * The adaptor functionality is limited to the following:
 * - data bits: 7 or 8
 * - stop bits: 1 or 2
 * - parity: even, odd or none
 * - flow control: none
 * - baud rates: 0, 2400, 4800, 9600, 19200, 38400, 57600, 115200, 153600
 * - signals: DTR, RTS and BREAK
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/tty.h>
#include <linux/usb.h>
#include <linux/usb/serial.h>

#define DRIVER_DESC "Renesas uPD78F0730 USB to serial converter driver"

#define DRIVER_AUTHOR "Maksim Salau <maksim.salau@gmail.com>"

static const struct usb_device_id id_table[] = {
        { USB_DEVICE(0x0409, 0x0063) }, /* V850ESJX3-STICK */
        { USB_DEVICE(0x045B, 0x0212) }, /* YRPBRL78G13, YRPBRL78G14 */
        { USB_DEVICE(0x064B, 0x7825) }, /* Analog Devices EVAL-ADXL362Z-DB */
        {}
};

MODULE_DEVICE_TABLE(usb, id_table);

/*
 * Each adaptor is associated with a private structure, that holds the current
 * state of control signals (DTR, RTS and BREAK).
 */
struct upd78f0730_port_private {
        struct mutex        lock;                /* mutex to protect line_signals */
        u8                line_signals;
};

/* Op-codes of control commands */
#define UPD78F0730_CMD_LINE_CONTROL        0x00
#define UPD78F0730_CMD_SET_DTR_RTS        0x01
#define UPD78F0730_CMD_SET_XON_XOFF_CHR        0x02
#define UPD78F0730_CMD_OPEN_CLOSE        0x03
#define UPD78F0730_CMD_SET_ERR_CHR        0x04

/* Data sizes in UPD78F0730_CMD_LINE_CONTROL command */
#define UPD78F0730_DATA_SIZE_7_BITS        0x00
#define UPD78F0730_DATA_SIZE_8_BITS        0x01
#define UPD78F0730_DATA_SIZE_MASK        0x01

/* Stop-bit modes in UPD78F0730_CMD_LINE_CONTROL command */
#define UPD78F0730_STOP_BIT_1_BIT        0x00
#define UPD78F0730_STOP_BIT_2_BIT        0x02
#define UPD78F0730_STOP_BIT_MASK        0x02

/* Parity modes in UPD78F0730_CMD_LINE_CONTROL command */
#define UPD78F0730_PARITY_NONE        0x00
#define UPD78F0730_PARITY_EVEN        0x04
#define UPD78F0730_PARITY_ODD        0x08
#define UPD78F0730_PARITY_MASK        0x0C

/* Flow control modes in UPD78F0730_CMD_LINE_CONTROL command */
#define UPD78F0730_FLOW_CONTROL_NONE        0x00
#define UPD78F0730_FLOW_CONTROL_HW        0x10
#define UPD78F0730_FLOW_CONTROL_SW        0x20
#define UPD78F0730_FLOW_CONTROL_MASK        0x30

/* Control signal bits in UPD78F0730_CMD_SET_DTR_RTS command */
#define UPD78F0730_RTS                0x01
#define UPD78F0730_DTR                0x02
#define UPD78F0730_BREAK        0x04

/* Port modes in UPD78F0730_CMD_OPEN_CLOSE command */
#define UPD78F0730_PORT_CLOSE        0x00
#define UPD78F0730_PORT_OPEN        0x01

/* Error character substitution modes in UPD78F0730_CMD_SET_ERR_CHR command */
#define UPD78F0730_ERR_CHR_DISABLED        0x00
#define UPD78F0730_ERR_CHR_ENABLED        0x01

/*
 * Declaration of command structures
 */

/* UPD78F0730_CMD_LINE_CONTROL command */
struct upd78f0730_line_control {
        u8        opcode;
        __le32        baud_rate;
        u8        params;
} __packed;

/* UPD78F0730_CMD_SET_DTR_RTS command */
struct upd78f0730_set_dtr_rts {
        u8 opcode;
        u8 params;
};

/* UPD78F0730_CMD_SET_XON_OFF_CHR command */
struct upd78f0730_set_xon_xoff_chr {
        u8 opcode;
        u8 xon;
        u8 xoff;
};

/* UPD78F0730_CMD_OPEN_CLOSE command */
struct upd78f0730_open_close {
        u8 opcode;
        u8 state;
};

/* UPD78F0730_CMD_SET_ERR_CHR command */
struct upd78f0730_set_err_chr {
        u8 opcode;
        u8 state;
        u8 err_char;
};

static int upd78f0730_send_ctl(struct usb_serial_port *port,
                        const void *data, int size)
{
        struct usb_device *usbdev = port->serial->dev;
        void *buf;
        int res;

        if (size <= 0 || !data)
                return -EINVAL;

        buf = kmemdup(data, size, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;

        res = usb_control_msg(usbdev, usb_sndctrlpipe(usbdev, 0), 0x00,
                        USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT,
                        0x0000, 0x0000, buf, size, USB_CTRL_SET_TIMEOUT);

        kfree(buf);

        if (res < 0) {
                struct device *dev = &port->dev;

                dev_err(dev, "failed to send control request %02x: %d\n",
                        *(u8 *)data, res);

                return res;
        }

        return 0;
}

static int upd78f0730_port_probe(struct usb_serial_port *port)
{
        struct upd78f0730_port_private *private;

        private = kzalloc(sizeof(*private), GFP_KERNEL);
        if (!private)
                return -ENOMEM;

        mutex_init(&private->lock);
        usb_set_serial_port_data(port, private);

        return 0;
}

static void upd78f0730_port_remove(struct usb_serial_port *port)
{
        struct upd78f0730_port_private *private;

        private = usb_get_serial_port_data(port);
        mutex_destroy(&private->lock);
        kfree(private);
}

static int upd78f0730_tiocmget(struct tty_struct *tty)
{
        struct upd78f0730_port_private *private;
        struct usb_serial_port *port = tty->driver_data;
        int signals;
        int res;

        private = usb_get_serial_port_data(port);

        mutex_lock(&private->lock);
        signals = private->line_signals;
        mutex_unlock(&private->lock);

        res = ((signals & UPD78F0730_DTR) ? TIOCM_DTR : 0) |
                ((signals & UPD78F0730_RTS) ? TIOCM_RTS : 0);

        dev_dbg(&port->dev, "%s - res = %x\n", __func__, res);

        return res;
}

static int upd78f0730_tiocmset(struct tty_struct *tty,
                        unsigned int set, unsigned int clear)
{
        struct usb_serial_port *port = tty->driver_data;
        struct upd78f0730_port_private *private;
        struct upd78f0730_set_dtr_rts request;
        struct device *dev = &port->dev;
        int res;

        private = usb_get_serial_port_data(port);

        mutex_lock(&private->lock);
        if (set & TIOCM_DTR) {
                private->line_signals |= UPD78F0730_DTR;
                dev_dbg(dev, "%s - set DTR\n", __func__);
        }
        if (set & TIOCM_RTS) {
                private->line_signals |= UPD78F0730_RTS;
                dev_dbg(dev, "%s - set RTS\n", __func__);
        }
        if (clear & TIOCM_DTR) {
                private->line_signals &= ~UPD78F0730_DTR;
                dev_dbg(dev, "%s - clear DTR\n", __func__);
        }
        if (clear & TIOCM_RTS) {
                private->line_signals &= ~UPD78F0730_RTS;
                dev_dbg(dev, "%s - clear RTS\n", __func__);
        }
        request.opcode = UPD78F0730_CMD_SET_DTR_RTS;
        request.params = private->line_signals;

        res = upd78f0730_send_ctl(port, &request, sizeof(request));
        mutex_unlock(&private->lock);

        return res;
}

static int upd78f0730_break_ctl(struct tty_struct *tty, int break_state)
{
        struct upd78f0730_port_private *private;
        struct usb_serial_port *port = tty->driver_data;
        struct upd78f0730_set_dtr_rts request;
        struct device *dev = &port->dev;
        int res;

        private = usb_get_serial_port_data(port);

        mutex_lock(&private->lock);
        if (break_state) {
                private->line_signals |= UPD78F0730_BREAK;
                dev_dbg(dev, "%s - set BREAK\n", __func__);
        } else {
                private->line_signals &= ~UPD78F0730_BREAK;
                dev_dbg(dev, "%s - clear BREAK\n", __func__);
        }
        request.opcode = UPD78F0730_CMD_SET_DTR_RTS;
        request.params = private->line_signals;

        res = upd78f0730_send_ctl(port, &request, sizeof(request));
        mutex_unlock(&private->lock);

        return res;
}

static void upd78f0730_dtr_rts(struct usb_serial_port *port, int on)
{
        struct tty_struct *tty = port->port.tty;
        unsigned int set = 0;
        unsigned int clear = 0;

        if (on)
                set = TIOCM_DTR | TIOCM_RTS;
        else
                clear = TIOCM_DTR | TIOCM_RTS;

        upd78f0730_tiocmset(tty, set, clear);
}

static speed_t upd78f0730_get_baud_rate(struct tty_struct *tty)
{
        const speed_t baud_rate = tty_get_baud_rate(tty);
        static const speed_t supported[] = {
                0, 2400, 4800, 9600, 19200, 38400, 57600, 115200, 153600
        };
        int i;

        for (i = ARRAY_SIZE(supported) - 1; i >= 0; i--) {
                if (baud_rate == supported[i])
                        return baud_rate;
        }

        /* If the baud rate is not supported, switch to the default one */
        tty_encode_baud_rate(tty, 9600, 9600);

        return tty_get_baud_rate(tty);
}

static void upd78f0730_set_termios(struct tty_struct *tty,
                                   struct usb_serial_port *port,
                                   const struct ktermios *old_termios)
{
        struct device *dev = &port->dev;
        struct upd78f0730_line_control request;
        speed_t baud_rate;

        if (old_termios && !tty_termios_hw_change(&tty->termios, old_termios))
                return;

        if (C_BAUD(tty) == B0)
                upd78f0730_dtr_rts(port, 0);
        else if (old_termios && (old_termios->c_cflag & CBAUD) == B0)
                upd78f0730_dtr_rts(port, 1);

        baud_rate = upd78f0730_get_baud_rate(tty);
        request.opcode = UPD78F0730_CMD_LINE_CONTROL;
        request.baud_rate = cpu_to_le32(baud_rate);
        request.params = 0;
        dev_dbg(dev, "%s - baud rate = %d\n", __func__, baud_rate);

        switch (C_CSIZE(tty)) {
        case CS7:
                request.params |= UPD78F0730_DATA_SIZE_7_BITS;
                dev_dbg(dev, "%s - 7 data bits\n", __func__);
                break;
        default:
                tty->termios.c_cflag &= ~CSIZE;
                tty->termios.c_cflag |= CS8;
                dev_warn(dev, "data size is not supported, using 8 bits\n");
                fallthrough;
        case CS8:
                request.params |= UPD78F0730_DATA_SIZE_8_BITS;
                dev_dbg(dev, "%s - 8 data bits\n", __func__);
                break;
        }

        if (C_PARENB(tty)) {
                if (C_PARODD(tty)) {
                        request.params |= UPD78F0730_PARITY_ODD;
                        dev_dbg(dev, "%s - odd parity\n", __func__);
                } else {
                        request.params |= UPD78F0730_PARITY_EVEN;
                        dev_dbg(dev, "%s - even parity\n", __func__);
                }

                if (C_CMSPAR(tty)) {
                        tty->termios.c_cflag &= ~CMSPAR;
                        dev_warn(dev, "MARK/SPACE parity is not supported\n");
                }
        } else {
                request.params |= UPD78F0730_PARITY_NONE;
                dev_dbg(dev, "%s - no parity\n", __func__);
        }

        if (C_CSTOPB(tty)) {
                request.params |= UPD78F0730_STOP_BIT_2_BIT;
                dev_dbg(dev, "%s - 2 stop bits\n", __func__);
        } else {
                request.params |= UPD78F0730_STOP_BIT_1_BIT;
                dev_dbg(dev, "%s - 1 stop bit\n", __func__);
        }

        if (C_CRTSCTS(tty)) {
                tty->termios.c_cflag &= ~CRTSCTS;
                dev_warn(dev, "RTSCTS flow control is not supported\n");
        }
        if (I_IXOFF(tty) || I_IXON(tty)) {
                tty->termios.c_iflag &= ~(IXOFF | IXON);
                dev_warn(dev, "XON/XOFF flow control is not supported\n");
        }
        request.params |= UPD78F0730_FLOW_CONTROL_NONE;
        dev_dbg(dev, "%s - no flow control\n", __func__);

        upd78f0730_send_ctl(port, &request, sizeof(request));
}

static int upd78f0730_open(struct tty_struct *tty, struct usb_serial_port *port)
{
        static const struct upd78f0730_open_close request = {
                .opcode = UPD78F0730_CMD_OPEN_CLOSE,
                .state = UPD78F0730_PORT_OPEN
        };
        int res;

        res = upd78f0730_send_ctl(port, &request, sizeof(request));
        if (res)
                return res;

        if (tty)
                upd78f0730_set_termios(tty, port, NULL);

        return usb_serial_generic_open(tty, port);
}

static void upd78f0730_close(struct usb_serial_port *port)
{
        static const struct upd78f0730_open_close request = {
                .opcode = UPD78F0730_CMD_OPEN_CLOSE,
                .state = UPD78F0730_PORT_CLOSE
        };

        usb_serial_generic_close(port);
        upd78f0730_send_ctl(port, &request, sizeof(request));
}

static struct usb_serial_driver upd78f0730_device = {
        .driver         = {
                .name        = "upd78f0730",
        },
        .id_table        = id_table,
        .num_ports        = 1,
        .port_probe        = upd78f0730_port_probe,
        .port_remove        = upd78f0730_port_remove,
        .open                = upd78f0730_open,
        .close                = upd78f0730_close,
        .set_termios        = upd78f0730_set_termios,
        .tiocmget        = upd78f0730_tiocmget,
        .tiocmset        = upd78f0730_tiocmset,
        .dtr_rts        = upd78f0730_dtr_rts,
        .break_ctl        = upd78f0730_break_ctl,
};

static struct usb_serial_driver * const serial_drivers[] = {
        &upd78f0730_device,
        NULL
};

module_usb_serial_driver(serial_drivers, id_table);

MODULE_DESCRIPTION(DRIVER_DESC);
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_LICENSE("GPL v2");





































































































































































































































































































































































































































































































































































































































































































   82 




































































































































































































































































































































































































































































  665 

























































































































































































































































































































































   60 










    3 





























































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* SCTP kernel implementation
 * (C) Copyright IBM Corp. 2001, 2004
 * Copyright (c) 1999-2000 Cisco, Inc.
 * Copyright (c) 1999-2001 Motorola, Inc.
 * Copyright (c) 2001 Intel Corp.
 *
 * This file is part of the SCTP kernel implementation
 *
 * Please send any bug reports or fixes you make to the
 * email addresses:
 *    lksctp developers <linux-sctp@vger.kernel.org>
 *
 * Written or modified by:
 *    Randall Stewart            <randall@sctp.chicago.il.us>
 *    Ken Morneau            <kmorneau@cisco.com>
 *    Qiaobing Xie            <qxie1@email.mot.com>
 *    La Monte H.P. Yarroll <piggy@acm.org>
 *    Karl Knutson            <karl@athena.chicago.il.us>
 *    Jon Grimm                    <jgrimm@us.ibm.com>
 *    Xingang Guo            <xingang.guo@intel.com>
 *    Hui Huang                    <hui.huang@nokia.com>
 *    Sridhar Samudrala            <sri@us.ibm.com>
 *    Daisy Chang            <daisyc@us.ibm.com>
 *    Dajiang Zhang            <dajiang.zhang@nokia.com>
 *    Ardelle Fan            <ardelle.fan@intel.com>
 *    Ryan Layer            <rmlayer@us.ibm.com>
 *    Anup Pemmaiah            <pemmaiah@cc.usu.edu>
 *    Kevin Gao             <kevin.gao@intel.com>
 */

#ifndef __sctp_structs_h__
#define __sctp_structs_h__

#include <crypto/sha2.h>
#include <linux/ktime.h>
#include <linux/generic-radix-tree.h>
#include <linux/rhashtable-types.h>
#include <linux/socket.h>        /* linux/in.h needs this!!    */
#include <linux/in.h>                /* We get struct sockaddr_in. */
#include <linux/in6.h>                /* We get struct in6_addr     */
#include <linux/ipv6.h>
#include <asm/param.h>                /* We get MAXHOSTNAMELEN.     */
#include <linux/atomic.h>                /* This gets us atomic counters.  */
#include <linux/skbuff.h>        /* We need sk_buff_head. */
#include <linux/workqueue.h>        /* We need tq_struct.         */
#include <linux/sctp.h>                /* We need sctp* header structs.  */
#include <net/sctp/auth.h>        /* We need auth specific structs */
#include <net/ip.h>                /* For inet_skb_parm */

/* A convenience structure for handling sockaddr structures.
 * We should wean ourselves off this.
 */
union sctp_addr {
        struct sockaddr_inet sa;        /* Large enough for both address families */
        struct sockaddr_in v4;
        struct sockaddr_in6 v6;
};

/* Forward declarations for data structures. */
struct sctp_globals;
struct sctp_endpoint;
struct sctp_association;
struct sctp_transport;
struct sctp_packet;
struct sctp_chunk;
struct sctp_inq;
struct sctp_outq;
struct sctp_bind_addr;
struct sctp_ulpq;
struct sctp_ep_common;
struct sctp_stream;


#include <net/sctp/tsnmap.h>
#include <net/sctp/ulpevent.h>
#include <net/sctp/ulpqueue.h>
#include <net/sctp/stream_interleave.h>

/* Structures useful for managing bind/connect. */

struct sctp_bind_bucket {
        unsigned short        port;
        signed char        fastreuse;
        signed char        fastreuseport;
        kuid_t                fastuid;
        struct hlist_node        node;
        struct hlist_head        owner;
        struct net        *net;
};

struct sctp_bind_hashbucket {
        spinlock_t        lock;
        struct hlist_head        chain;
};

/* Used for hashing all associations.  */
struct sctp_hashbucket {
        rwlock_t        lock;
        struct hlist_head        chain;
} __attribute__((__aligned__(8)));


/* The SCTP globals structure. */
extern struct sctp_globals {
        /* This is a list of groups of functions for each address
         * family that we support.
         */
        struct list_head address_families;

        /* This is the hash of all endpoints. */
        struct sctp_hashbucket *ep_hashtable;
        /* This is the sctp port control hash.        */
        struct sctp_bind_hashbucket *port_hashtable;
        /* This is the hash of all transports. */
        struct rhltable transport_hashtable;

        /* Sizes of above hashtables. */
        int ep_hashsize;
        int port_hashsize;

        /* Default initialization values to be applied to new associations. */
        __u16 max_instreams;
        __u16 max_outstreams;

        /* Flag to indicate whether computing and verifying checksum
         * is disabled. */
        bool checksum_disable;
} sctp_globals;

#define sctp_max_instreams                (sctp_globals.max_instreams)
#define sctp_max_outstreams                (sctp_globals.max_outstreams)
#define sctp_address_families                (sctp_globals.address_families)
#define sctp_ep_hashsize                (sctp_globals.ep_hashsize)
#define sctp_ep_hashtable                (sctp_globals.ep_hashtable)
#define sctp_port_hashsize                (sctp_globals.port_hashsize)
#define sctp_port_hashtable                (sctp_globals.port_hashtable)
#define sctp_transport_hashtable        (sctp_globals.transport_hashtable)
#define sctp_checksum_disable                (sctp_globals.checksum_disable)

/* SCTP Socket type: UDP or TCP style. */
enum sctp_socket_type {
        SCTP_SOCKET_UDP = 0,
        SCTP_SOCKET_UDP_HIGH_BANDWIDTH,
        SCTP_SOCKET_TCP
};

/* Per socket SCTP information. */
struct sctp_sock {
        /* inet_sock has to be the first member of sctp_sock */
        struct inet_sock inet;
        /* What kind of a socket is this? */
        enum sctp_socket_type type;

        /* PF_ family specific functions.  */
        struct sctp_pf *pf;

        /* What is our base endpointer? */
        struct sctp_endpoint *ep;

        struct sctp_bind_bucket *bind_hash;
        /* Various Socket Options.  */
        __u16 default_stream;
        __u32 default_ppid;
        __u16 default_flags;
        __u32 default_context;
        __u32 default_timetolive;
        __u32 default_rcv_context;
        int max_burst;

        /* Heartbeat interval: The endpoint sends out a Heartbeat chunk to
         * the destination address every heartbeat interval. This value
         * will be inherited by all new associations.
         */
        __u32 hbinterval;
        __u32 probe_interval;

        __be16 udp_port;
        __be16 encap_port;

        /* This is the max_retrans value for new associations. */
        __u16 pathmaxrxt;

        __u32 flowlabel;
        __u8  dscp;

        __u16 pf_retrans;
        __u16 ps_retrans;

        /* The initial Path MTU to use for new associations. */
        __u32 pathmtu;

        /* The default SACK delay timeout for new associations. */
        __u32 sackdelay;
        __u32 sackfreq;

        /* Flags controlling Heartbeat, SACK delay, and Path MTU Discovery. */
        __u32 param_flags;

        __u32 default_ss;

        struct sctp_rtoinfo rtoinfo;
        struct sctp_paddrparams paddrparam;
        struct sctp_assocparams assocparams;

        /*
         * These two structures must be grouped together for the usercopy
         * whitelist region.
         */
        __u16 subscribe;
        struct sctp_initmsg initmsg;

        int user_frag;

        __u32 autoclose;
        __u32 adaptation_ind;
        __u32 pd_point;
        __u16        nodelay:1,
                pf_expose:2,
                reuse:1,
                disable_fragments:1,
                v4mapped:1,
                frag_interleave:1,
                recvrcvinfo:1,
                recvnxtinfo:1,
                data_ready_signalled:1,
                cookie_auth_enable:1;

        atomic_t pd_mode;

        /* Fields after this point will be skipped on copies, like on accept
         * and peeloff operations
         */

        /* Receive to here while partial delivery is in effect. */
        struct sk_buff_head pd_lobby;

        struct list_head auto_asconf_list;
        int do_auto_asconf;
};

#define sctp_sk(ptr) container_of_const(ptr, struct sctp_sock, inet.sk)

static inline struct sock *sctp_opt2sk(const struct sctp_sock *sp)
{
       return (struct sock *)sp;
}

#if IS_ENABLED(CONFIG_IPV6)
struct sctp6_sock {
       struct sctp_sock  sctp;
       struct ipv6_pinfo inet6;
};
#endif /* CONFIG_IPV6 */


/* This is our APPLICATION-SPECIFIC state cookie.
 * THIS IS NOT DICTATED BY THE SPECIFICATION.
 */
/* These are the parts of an association which we send in the cookie.
 * Most of these are straight out of:
 * RFC2960 12.2 Parameters necessary per association (i.e. the TCB)
 *
 */

struct sctp_cookie {

        /* My               : Tag expected in every inbound packet and sent
         * Verification: in the INIT or INIT ACK chunk.
         * Tag               :
         */
        __u32 my_vtag;

        /* Peer's      : Tag expected in every outbound packet except
         * Verification: in the INIT chunk.
         * Tag               :
         */
        __u32 peer_vtag;

        /* The rest of these are not from the spec, but really need to
         * be in the cookie.
         */

        /* My Tie Tag  : Assist in discovering a restarting association. */
        __u32 my_ttag;

        /* Peer's Tie Tag: Assist in discovering a restarting association. */
        __u32 peer_ttag;

        /* When does this cookie expire? */
        ktime_t expiration;

        /* Number of inbound/outbound streams which are set
         * and negotiated during the INIT process.
         */
        __u16 sinit_num_ostreams;
        __u16 sinit_max_instreams;

        /* This is the first sequence number I used.  */
        __u32 initial_tsn;

        /* This holds the originating address of the INIT packet.  */
        union sctp_addr peer_addr;

        /* IG Section 2.35.3 
         * Include the source port of the INIT-ACK
         */
        __u16                my_port;

        __u8 prsctp_capable;

        /* Padding for future use */
        __u8 padding;                  

        __u32 adaptation_ind;

        __u8 auth_random[sizeof(struct sctp_paramhdr) +
                         SCTP_AUTH_RANDOM_LENGTH];
        __u8 auth_hmacs[SCTP_AUTH_NUM_HMACS * sizeof(__u16) + 2];
        __u8 auth_chunks[sizeof(struct sctp_paramhdr) + SCTP_AUTH_MAX_CHUNKS];

        /* This is a shim for my peer's INIT packet, followed by
         * a copy of the raw address list of the association.
         * The length of the raw address list is saved in the
         * raw_addr_list_len field, which will be used at the time when
         * the association TCB is re-constructed from the cookie.
         */
        __u32 raw_addr_list_len;
        /* struct sctp_init_chunk peer_init[]; */
};


/* The format of our cookie that we send to our peer. */
struct sctp_signed_cookie {
        __u8 mac[SCTP_COOKIE_MAC_SIZE];
        __u32 __pad;                /* force sctp_cookie alignment to 64 bits */
        struct sctp_cookie c;
} __packed;

/* This is another convenience type to allocate memory for address
 * params for the maximum size and pass such structures around
 * internally.
 */
union sctp_addr_param {
        struct sctp_paramhdr p;
        struct sctp_ipv4addr_param v4;
        struct sctp_ipv6addr_param v6;
};

/* A convenience type to allow walking through the various
 * parameters and avoid casting all over the place.
 */
union sctp_params {
        void *v;
        struct sctp_paramhdr *p;
        struct sctp_cookie_preserve_param *life;
        struct sctp_hostname_param *dns;
        struct sctp_cookie_param *cookie;
        struct sctp_supported_addrs_param *sat;
        struct sctp_ipv4addr_param *v4;
        struct sctp_ipv6addr_param *v6;
        union sctp_addr_param *addr;
        struct sctp_adaptation_ind_param *aind;
        struct sctp_supported_ext_param *ext;
        struct sctp_random_param *random;
        struct sctp_chunks_param *chunks;
        struct sctp_hmac_algo_param *hmac_algo;
        struct sctp_addip_param *addip;
};

/* RFC 2960.  Section 3.3.5 Heartbeat.
 *    Heartbeat Information: variable length
 *    The Sender-specific Heartbeat Info field should normally include
 *    information about the sender's current time when this HEARTBEAT
 *    chunk is sent and the destination transport address to which this
 *    HEARTBEAT is sent (see Section 8.3).
 */
struct sctp_sender_hb_info {
        struct sctp_paramhdr param_hdr;
        union sctp_addr daddr;
        unsigned long sent_at;
        __u64 hb_nonce;
        __u32 probe_size;
};

int sctp_stream_init(struct sctp_stream *stream, __u16 outcnt, __u16 incnt,
                     gfp_t gfp);
int sctp_stream_init_ext(struct sctp_stream *stream, __u16 sid);
void sctp_stream_free(struct sctp_stream *stream);
void sctp_stream_clear(struct sctp_stream *stream);
void sctp_stream_update(struct sctp_stream *stream, struct sctp_stream *new);

/* What is the current SSN number for this stream? */
#define sctp_ssn_peek(stream, type, sid) \
        (sctp_stream_##type((stream), (sid))->ssn)

/* Return the next SSN number for this stream.        */
#define sctp_ssn_next(stream, type, sid) \
        (sctp_stream_##type((stream), (sid))->ssn++)

/* Skip over this ssn and all below. */
#define sctp_ssn_skip(stream, type, sid, ssn) \
        (sctp_stream_##type((stream), (sid))->ssn = ssn + 1)

/* What is the current MID number for this stream? */
#define sctp_mid_peek(stream, type, sid) \
        (sctp_stream_##type((stream), (sid))->mid)

/* Return the next MID number for this stream.  */
#define sctp_mid_next(stream, type, sid) \
        (sctp_stream_##type((stream), (sid))->mid++)

/* Skip over this mid and all below. */
#define sctp_mid_skip(stream, type, sid, mid) \
        (sctp_stream_##type((stream), (sid))->mid = mid + 1)

/* What is the current MID_uo number for this stream? */
#define sctp_mid_uo_peek(stream, type, sid) \
        (sctp_stream_##type((stream), (sid))->mid_uo)

/* Return the next MID_uo number for this stream.  */
#define sctp_mid_uo_next(stream, type, sid) \
        (sctp_stream_##type((stream), (sid))->mid_uo++)

/*
 * Pointers to address related SCTP functions.
 * (i.e. things that depend on the address family.)
 */
struct sctp_af {
        int                (*sctp_xmit)        (struct sk_buff *skb,
                                         struct sctp_transport *);
        int                (*setsockopt)        (struct sock *sk,
                                         int level,
                                         int optname,
                                         sockptr_t optval,
                                         unsigned int optlen);
        int                (*getsockopt)        (struct sock *sk,
                                         int level,
                                         int optname,
                                         char __user *optval,
                                         int __user *optlen);
        void                (*get_dst)        (struct sctp_transport *t,
                                         union sctp_addr *saddr,
                                         struct flowi *fl,
                                         struct sock *sk);
        void                (*get_saddr)        (struct sctp_sock *sk,
                                         struct sctp_transport *t,
                                         struct flowi *fl);
        void                (*copy_addrlist) (struct list_head *,
                                          struct net_device *);
        int                (*cmp_addr)        (const union sctp_addr *addr1,
                                         const union sctp_addr *addr2);
        void                (*addr_copy)        (union sctp_addr *dst,
                                         union sctp_addr *src);
        void                (*from_skb)        (union sctp_addr *,
                                         struct sk_buff *skb,
                                         int saddr);
        void                (*from_sk)        (union sctp_addr *,
                                         struct sock *sk);
        bool                (*from_addr_param) (union sctp_addr *,
                                            union sctp_addr_param *,
                                            __be16 port, int iif);
        int                (*to_addr_param) (const union sctp_addr *,
                                          union sctp_addr_param *); 
        int                (*addr_valid)        (union sctp_addr *,
                                         struct sctp_sock *,
                                         const struct sk_buff *);
        enum sctp_scope        (*scope)(union sctp_addr *);
        void                (*inaddr_any)        (union sctp_addr *, __be16);
        int                (*is_any)        (const union sctp_addr *);
        int                (*available)        (union sctp_addr *,
                                         struct sctp_sock *);
        int                (*skb_iif)        (const struct sk_buff *sk);
        int                (*skb_sdif)(const struct sk_buff *sk);
        int                (*is_ce)        (const struct sk_buff *sk);
        void                (*seq_dump_addr)(struct seq_file *seq,
                                         union sctp_addr *addr);
        void                (*ecn_capable)(struct sock *sk);
        __u16                net_header_len;
        int                sockaddr_len;
        int                (*ip_options_len)(struct sock *sk);
        sa_family_t        sa_family;
        struct list_head list;
};

struct sctp_af *sctp_get_af_specific(sa_family_t);
int sctp_register_af(struct sctp_af *);

/* Protocol family functions. */
struct sctp_pf {
        void (*event_msgname)(struct sctp_ulpevent *, char *, int *);
        void (*skb_msgname)  (struct sk_buff *, char *, int *);
        int  (*af_supported) (sa_family_t, struct sctp_sock *);
        int  (*cmp_addr) (const union sctp_addr *,
                          const union sctp_addr *,
                          struct sctp_sock *);
        int  (*bind_verify) (struct sctp_sock *, union sctp_addr *);
        int  (*send_verify) (struct sctp_sock *, union sctp_addr *);
        int  (*supported_addrs)(const struct sctp_sock *, __be16 *);
        struct sock *(*create_accept_sk) (struct sock *sk,
                                          struct sctp_association *asoc,
                                          bool kern);
        int (*addr_to_user)(struct sctp_sock *sk, union sctp_addr *addr);
        void (*to_sk_saddr)(union sctp_addr *, struct sock *sk);
        void (*to_sk_daddr)(union sctp_addr *, struct sock *sk);
        void (*copy_ip_options)(struct sock *sk, struct sock *newsk);
        struct sctp_af *af;
};


/* Structure to track chunk fragments that have been acked, but peer
 * fragments of the same message have not.
 */
struct sctp_datamsg {
        /* Chunks waiting to be submitted to lower layer. */
        struct list_head chunks;
        /* Reference counting. */
        refcount_t refcnt;
        /* When is this message no longer interesting to the peer? */
        unsigned long expires_at;
        /* Did the message fail to send? */
        int send_error;
        u8 send_failed:1,
           can_delay:1,        /* should this message be Nagle delayed */
           abandoned:1;        /* should this message be abandoned */
};

struct sctp_datamsg *sctp_datamsg_from_user(struct sctp_association *,
                                            struct sctp_sndrcvinfo *,
                                            struct iov_iter *);
void sctp_datamsg_free(struct sctp_datamsg *);
void sctp_datamsg_put(struct sctp_datamsg *);
void sctp_chunk_fail(struct sctp_chunk *, int error);
int sctp_chunk_abandoned(struct sctp_chunk *);

/* RFC2960 1.4 Key Terms
 *
 * o Chunk: A unit of information within an SCTP packet, consisting of
 * a chunk header and chunk-specific content.
 *
 * As a matter of convenience, we remember the SCTP common header for
 * each chunk as well as a few other header pointers...
 */
struct sctp_chunk {
        struct list_head list;

        refcount_t refcnt;

        /* How many times this chunk have been sent, for prsctp RTX policy */
        int sent_count;

        union {
                /* This is our link to the per-transport transmitted list.  */
                struct list_head transmitted_list;
                /* List in specific stream outq */
                struct list_head stream_list;
        };

        /* This field is used by chunks that hold fragmented data.
         * For the first fragment this is the list that holds the rest of
         * fragments. For the remaining fragments, this is the link to the
         * frag_list maintained in the first fragment.
         */
        struct list_head frag_list;

        /* This points to the sk_buff containing the actual data.  */
        struct sk_buff *skb;

        union {
                /* In case of GSO packets, this will store the head one */
                struct sk_buff *head_skb;
                /* In case of auth enabled, this will point to the shkey */
                struct sctp_shared_key *shkey;
        };

        /* These are the SCTP headers by reverse order in a packet.
         * Note that some of these may happen more than once.  In that
         * case, we point at the "current" one, whatever that means
         * for that level of header.
         */

        /* We point this at the FIRST TLV parameter to chunk_hdr.  */
        union sctp_params param_hdr;
        union {
                __u8 *v;
                struct sctp_datahdr *data_hdr;
                struct sctp_inithdr *init_hdr;
                struct sctp_sackhdr *sack_hdr;
                struct sctp_heartbeathdr *hb_hdr;
                struct sctp_sender_hb_info *hbs_hdr;
                struct sctp_shutdownhdr *shutdown_hdr;
                struct sctp_signed_cookie *cookie_hdr;
                struct sctp_ecnehdr *ecne_hdr;
                struct sctp_cwrhdr *ecn_cwr_hdr;
                struct sctp_errhdr *err_hdr;
                struct sctp_addiphdr *addip_hdr;
                struct sctp_fwdtsn_hdr *fwdtsn_hdr;
                struct sctp_authhdr *auth_hdr;
                struct sctp_idatahdr *idata_hdr;
                struct sctp_ifwdtsn_hdr *ifwdtsn_hdr;
        } subh;

        __u8 *chunk_end;

        struct sctp_chunkhdr *chunk_hdr;
        struct sctphdr *sctp_hdr;

        /* This needs to be recoverable for SCTP_SEND_FAILED events. */
        struct sctp_sndrcvinfo sinfo;

        /* Which association does this belong to?  */
        struct sctp_association *asoc;

        /* What endpoint received this chunk? */
        struct sctp_ep_common *rcvr;

        /* We fill this in if we are calculating RTT. */
        unsigned long sent_at;

        /* What is the origin IP address for this chunk?  */
        union sctp_addr source;
        /* Destination address for this chunk. */
        union sctp_addr dest;

        /* For outbound message, track all fragments for SEND_FAILED. */
        struct sctp_datamsg *msg;

        /* For an inbound chunk, this tells us where it came from.
         * For an outbound chunk, it tells us where we'd like it to
         * go.        It is NULL if we have no preference.
         */
        struct sctp_transport *transport;

        /* SCTP-AUTH:  For the special case inbound processing of COOKIE-ECHO
         * we need save a pointer to the AUTH chunk, since the SCTP-AUTH
         * spec violates the principle premis that all chunks are processed
         * in order.
         */
        struct sk_buff *auth_chunk;

#define SCTP_CAN_FRTX 0x0
#define SCTP_NEED_FRTX 0x1
#define SCTP_DONT_FRTX 0x2
        __u16        rtt_in_progress:1,        /* This chunk used for RTT calc? */
                has_tsn:1,                /* Does this chunk have a TSN yet? */
                has_ssn:1,                /* Does this chunk have a SSN yet? */
#define has_mid has_ssn
                singleton:1,                /* Only chunk in the packet? */
                end_of_packet:1,        /* Last chunk in the packet? */
                ecn_ce_done:1,                /* Have we processed the ECN CE bit? */
                pdiscard:1,                /* Discard the whole packet now? */
                tsn_gap_acked:1,        /* Is this chunk acked by a GAP ACK? */
                data_accepted:1,        /* At least 1 chunk accepted */
                auth:1,                        /* IN: was auth'ed | OUT: needs auth */
                has_asconf:1,                /* IN: have seen an asconf before */
                pmtu_probe:1,                /* Used by PLPMTUD, can be set in s HB chunk */
                tsn_missing_report:2,        /* Data chunk missing counter. */
                fast_retransmit:2;        /* Is this chunk fast retransmitted? */
};

#define sctp_chunk_retransmitted(chunk)        (chunk->sent_count > 1)
void sctp_chunk_hold(struct sctp_chunk *);
void sctp_chunk_put(struct sctp_chunk *);
int sctp_user_addto_chunk(struct sctp_chunk *chunk, int len,
                          struct iov_iter *from);
void sctp_chunk_free(struct sctp_chunk *);
void  *sctp_addto_chunk(struct sctp_chunk *, int len, const void *data);
struct sctp_chunk *sctp_chunkify(struct sk_buff *,
                                 const struct sctp_association *,
                                 struct sock *, gfp_t gfp);
void sctp_init_addrs(struct sctp_chunk *, union sctp_addr *,
                     union sctp_addr *);
const union sctp_addr *sctp_source(const struct sctp_chunk *chunk);

static inline __u16 sctp_chunk_stream_no(struct sctp_chunk *ch)
{
        return ntohs(ch->subh.data_hdr->stream);
}

enum {
        SCTP_ADDR_NEW,                /* new address added to assoc/ep */
        SCTP_ADDR_SRC,                /* address can be used as source */
        SCTP_ADDR_DEL,                /* address about to be deleted */
};

/* This is a structure for holding either an IPv6 or an IPv4 address.  */
struct sctp_sockaddr_entry {
        struct list_head list;
        struct rcu_head        rcu;
        union sctp_addr a;
        __u8 state;
        __u8 valid;
};

#define SCTP_ADDRESS_TICK_DELAY        500

/* This structure holds lists of chunks as we are assembling for
 * transmission.
 */
struct sctp_packet {
        /* These are the SCTP header values (host order) for the packet. */
        __u16 source_port;
        __u16 destination_port;
        __u32 vtag;

        /* This contains the payload chunks.  */
        struct list_head chunk_list;

        /* This is the overhead of the sctp and ip headers. */
        size_t overhead;
        /* This is the total size of all chunks INCLUDING padding.  */
        size_t size;
        /* This is the maximum size this packet may have */
        size_t max_size;

        /* The packet is destined for this transport address.
         * The function we finally use to pass down to the next lower
         * layer lives in the transport structure.
         */
        struct sctp_transport *transport;

        /* pointer to the auth chunk for this packet */
        struct sctp_chunk *auth;

        u8  has_cookie_echo:1,        /* This packet contains a COOKIE-ECHO chunk. */
            has_sack:1,                /* This packet contains a SACK chunk. */
            has_auth:1,                /* This packet contains an AUTH chunk */
            has_data:1,                /* This packet contains at least 1 DATA chunk */
            ipfragok:1;                /* So let ip fragment this packet */
};

void sctp_packet_init(struct sctp_packet *, struct sctp_transport *,
                      __u16 sport, __u16 dport);
void sctp_packet_config(struct sctp_packet *, __u32 vtag, int);
enum sctp_xmit sctp_packet_transmit_chunk(struct sctp_packet *packet,
                                          struct sctp_chunk *chunk,
                                          int one_packet, gfp_t gfp);
enum sctp_xmit sctp_packet_append_chunk(struct sctp_packet *packet,
                                        struct sctp_chunk *chunk);
int sctp_packet_transmit(struct sctp_packet *, gfp_t);
void sctp_packet_free(struct sctp_packet *);

static inline int sctp_packet_empty(struct sctp_packet *packet)
{
        return packet->size == packet->overhead;
}

/* This represents a remote transport address.
 * For local transport addresses, we just use union sctp_addr.
 *
 * RFC2960 Section 1.4 Key Terms
 *
 *   o        Transport address:  A Transport Address is traditionally defined
 *        by Network Layer address, Transport Layer protocol and Transport
 *        Layer port number.  In the case of SCTP running over IP, a
 *        transport address is defined by the combination of an IP address
 *        and an SCTP port number (where SCTP is the Transport protocol).
 *
 * RFC2960 Section 7.1 SCTP Differences from TCP Congestion control
 *
 *   o        The sender keeps a separate congestion control parameter set for
 *        each of the destination addresses it can send to (not each
 *        source-destination pair but for each destination).  The parameters
 *        should decay if the address is not used for a long enough time
 *        period.
 *
 */
struct sctp_transport {
        /* A list of transports. */
        struct list_head transports;
        struct rhlist_head node;

        /* Reference counting. */
        refcount_t refcnt;
        __u32        dead:1,
                /* RTO-Pending : A flag used to track if one of the DATA
                 *                chunks sent to this address is currently being
                 *                used to compute a RTT. If this flag is 0,
                 *                the next DATA chunk sent to this destination
                 *                should be used to compute a RTT and this flag
                 *                should be set. Every time the RTT
                 *                calculation completes (i.e. the DATA chunk
                 *                is SACK'd) clear this flag.
                 */
                rto_pending:1,

                /*
                 * hb_sent : a flag that signals that we have a pending
                 * heartbeat.
                 */
                hb_sent:1,

                /* Is the Path MTU update pending on this transport */
                pmtu_pending:1,

                dst_pending_confirm:1,        /* need to confirm neighbour */

                /* Has this transport moved the ctsn since we last sacked */
                sack_generation:1;
        u32 dst_cookie;

        struct flowi fl;

        /* This is the peer's IP address and port. */
        union sctp_addr ipaddr;

        /* These are the functions we call to handle LLP stuff.         */
        struct sctp_af *af_specific;

        /* Which association do we belong to?  */
        struct sctp_association *asoc;

        /* RFC2960
         *
         * 12.3 Per Transport Address Data
         *
         * For each destination transport address in the peer's
         * address list derived from the INIT or INIT ACK chunk, a
         * number of data elements needs to be maintained including:
         */
        /* RTO               : The current retransmission timeout value.  */
        unsigned long rto;

        __u32 rtt;                /* This is the most recent RTT.         */

        /* RTTVAR      : The current RTT variation.  */
        __u32 rttvar;

        /* SRTT               : The current smoothed round trip time.        */
        __u32 srtt;

        /*
         * These are the congestion stats.
         */
        /* cwnd               : The current congestion window.         */
        __u32 cwnd;                  /* This is the actual cwnd.  */

        /* ssthresh    : The current slow start threshold value.  */
        __u32 ssthresh;

        /* partial     : The tracking method for increase of cwnd when in
         * bytes acked : congestion avoidance mode (see Section 6.2.2)
         */
        __u32 partial_bytes_acked;

        /* Data that has been sent, but not acknowledged. */
        __u32 flight_size;

        __u32 burst_limited;        /* Holds old cwnd when max.burst is applied */

        /* Destination */
        struct dst_entry *dst;
        /* Source address. */
        union sctp_addr saddr;

        /* Heartbeat interval: The endpoint sends out a Heartbeat chunk to
         * the destination address every heartbeat interval.
         */
        unsigned long hbinterval;
        unsigned long probe_interval;

        /* SACK delay timeout */
        unsigned long sackdelay;
        __u32 sackfreq;

        atomic_t mtu_info;

        /* When was the last time that we heard from this transport? We use
         * this to pick new active and retran paths.
         */
        ktime_t last_time_heard;

        /* When was the last time that we sent a chunk using this
         * transport? We use this to check for idle transports
         */
        unsigned long last_time_sent;

        /* Last time(in jiffies) when cwnd is reduced due to the congestion
         * indication based on ECNE chunk.
         */
        unsigned long last_time_ecne_reduced;

        __be16 encap_port;

        /* This is the max_retrans value for the transport and will
         * be initialized from the assocs value.  This can be changed
         * using the SCTP_SET_PEER_ADDR_PARAMS socket option.
         */
        __u16 pathmaxrxt;

        __u32 flowlabel;
        __u8  dscp;

        /* This is the partially failed retrans value for the transport
         * and will be initialized from the assocs value.  This can be changed
         * using the SCTP_PEER_ADDR_THLDS socket option
         */
        __u16 pf_retrans;
        /* Used for primary path switchover. */
        __u16 ps_retrans;
        /* PMTU              : The current known path MTU.  */
        __u32 pathmtu;

        /* Flags controlling Heartbeat, SACK delay, and Path MTU Discovery. */
        __u32 param_flags;

        /* The number of times INIT has been sent on this transport. */
        int init_sent_count;

        /* state       : The current state of this destination,
         *             : i.e. SCTP_ACTIVE, SCTP_INACTIVE, SCTP_UNKNOWN.
         */
        int state;

        /* These are the error stats for this destination.  */

        /* Error count : The current error count for this destination.        */
        unsigned short error_count;

        /* Per               : A timer used by each destination.
         * Destination :
         * Timer       :
         *
         * [Everywhere else in the text this is called T3-rtx. -ed]
         */
        struct timer_list T3_rtx_timer;

        /* Heartbeat timer is per destination. */
        struct timer_list hb_timer;

        /* Timer to handle ICMP proto unreachable envets */
        struct timer_list proto_unreach_timer;

        /* Timer to handler reconf chunk rtx */
        struct timer_list reconf_timer;

        /* Timer to send a probe HB packet for PLPMTUD */
        struct timer_list probe_timer;

        /* Since we're using per-destination retransmission timers
         * (see above), we're also using per-destination "transmitted"
         * queues.  This probably ought to be a private struct
         * accessible only within the outqueue, but it's not, yet.
         */
        struct list_head transmitted;

        /* We build bundle-able packets for this transport here.  */
        struct sctp_packet packet;

        /* This is the list of transports that have chunks to send.  */
        struct list_head send_ready;

        /* State information saved for SFR_CACC algorithm. The key
         * idea in SFR_CACC is to maintain state at the sender on a
         * per-destination basis when a changeover happens.
         *        char changeover_active;
         *        char cycling_changeover;
         *        __u32 next_tsn_at_change;
         *        char cacc_saw_newack;
         */
        struct {
                /* An unsigned integer, which stores the next TSN to be
                 * used by the sender, at the moment of changeover.
                 */
                __u32 next_tsn_at_change;

                /* A flag which indicates the occurrence of a changeover */
                char changeover_active;

                /* A flag which indicates whether the change of primary is
                 * the first switch to this destination address during an
                 * active switch.
                 */
                char cycling_changeover;

                /* A temporary flag, which is used during the processing of
                 * a SACK to estimate the causative TSN(s)'s group.
                 */
                char cacc_saw_newack;
        } cacc;

        struct {
                __u16 pmtu;
                __u16 probe_size;
                __u16 probe_high;
                __u8 probe_count;
                __u8 state;
        } pl; /* plpmtud related */

        /* 64-bit random number sent with heartbeat. */
        __u64 hb_nonce;

        struct rcu_head rcu;
};

struct sctp_transport *sctp_transport_new(struct net *, const union sctp_addr *,
                                          gfp_t);
void sctp_transport_set_owner(struct sctp_transport *,
                              struct sctp_association *);
void sctp_transport_route(struct sctp_transport *, union sctp_addr *,
                          struct sctp_sock *);
void sctp_transport_pmtu(struct sctp_transport *, struct sock *sk);
void sctp_transport_free(struct sctp_transport *);
void sctp_transport_reset_t3_rtx(struct sctp_transport *);
void sctp_transport_reset_hb_timer(struct sctp_transport *);
void sctp_transport_reset_reconf_timer(struct sctp_transport *transport);
void sctp_transport_reset_probe_timer(struct sctp_transport *transport);
void sctp_transport_reset_raise_timer(struct sctp_transport *transport);
int sctp_transport_hold(struct sctp_transport *);
void sctp_transport_put(struct sctp_transport *);
void sctp_transport_update_rto(struct sctp_transport *, __u32);
void sctp_transport_raise_cwnd(struct sctp_transport *, __u32, __u32);
void sctp_transport_lower_cwnd(struct sctp_transport *t,
                               enum sctp_lower_cwnd reason);
void sctp_transport_burst_limited(struct sctp_transport *);
void sctp_transport_burst_reset(struct sctp_transport *);
unsigned long sctp_transport_timeout(struct sctp_transport *);
void sctp_transport_reset(struct sctp_transport *t);
bool sctp_transport_update_pmtu(struct sctp_transport *t, u32 pmtu);
void sctp_transport_immediate_rtx(struct sctp_transport *);
void sctp_transport_dst_release(struct sctp_transport *t);
void sctp_transport_dst_confirm(struct sctp_transport *t);
void sctp_transport_pl_send(struct sctp_transport *t);
bool sctp_transport_pl_recv(struct sctp_transport *t);


/* This is the structure we use to queue packets as they come into
 * SCTP.  We write packets to it and read chunks from it.
 */
struct sctp_inq {
        /* This is actually a queue of sctp_chunk each
         * containing a partially decoded packet.
         */
        struct list_head in_chunk_list;
        /* This is the packet which is currently off the in queue and is
         * being worked on through the inbound chunk processing.
         */
        struct sctp_chunk *in_progress;

        /* This is the delayed task to finish delivering inbound
         * messages.
         */
        struct work_struct immediate;
};

void sctp_inq_init(struct sctp_inq *);
void sctp_inq_free(struct sctp_inq *);
void sctp_inq_push(struct sctp_inq *, struct sctp_chunk *packet);
struct sctp_chunk *sctp_inq_pop(struct sctp_inq *);
struct sctp_chunkhdr *sctp_inq_peek(struct sctp_inq *);
void sctp_inq_set_th_handler(struct sctp_inq *, work_func_t);

/* This is the structure we use to hold outbound chunks.  You push
 * chunks in and they automatically pop out the other end as bundled
 * packets (it calls (*output_handler)()).
 *
 * This structure covers sections 6.3, 6.4, 6.7, 6.8, 6.10, 7., 8.1,
 * and 8.2 of the v13 draft.
 *
 * It handles retransmissions.        The connection to the timeout portion
 * of the state machine is through sctp_..._timeout() and timeout_handler.
 *
 * If you feed it SACKs, it will eat them.
 *
 * If you give it big chunks, it will fragment them.
 *
 * It assigns TSN's to data chunks.  This happens at the last possible
 * instant before transmission.
 *
 * When free()'d, it empties itself out via output_handler().
 */
struct sctp_outq {
        struct sctp_association *asoc;

        /* Data pending that has never been transmitted.  */
        struct list_head out_chunk_list;

        /* Stream scheduler being used */
        struct sctp_sched_ops *sched;

        unsigned int out_qlen;        /* Total length of queued data chunks. */

        /* Error of send failed, may used in SCTP_SEND_FAILED event. */
        unsigned int error;

        /* These are control chunks we want to send.  */
        struct list_head control_chunk_list;

        /* These are chunks that have been sacked but are above the
         * CTSN, or cumulative tsn ack point.
         */
        struct list_head sacked;

        /* Put chunks on this list to schedule them for
         * retransmission.
         */
        struct list_head retransmit;

        /* Put chunks on this list to save them for FWD TSN processing as
         * they were abandoned.
         */
        struct list_head abandoned;

        /* How many unackd bytes do we have in-flight?        */
        __u32 outstanding_bytes;

        /* Are we doing fast-rtx on this queue */
        char fast_rtx;

        /* Corked? */
        char cork;
};

void sctp_outq_init(struct sctp_association *, struct sctp_outq *);
void sctp_outq_teardown(struct sctp_outq *);
void sctp_outq_free(struct sctp_outq*);
void sctp_outq_tail(struct sctp_outq *, struct sctp_chunk *chunk, gfp_t);
int sctp_outq_sack(struct sctp_outq *, struct sctp_chunk *);
int sctp_outq_is_empty(const struct sctp_outq *);
void sctp_retransmit(struct sctp_outq *q, struct sctp_transport *transport,
                     enum sctp_retransmit_reason reason);
void sctp_retransmit_mark(struct sctp_outq *, struct sctp_transport *, __u8);
void sctp_outq_uncork(struct sctp_outq *, gfp_t gfp);
void sctp_prsctp_prune(struct sctp_association *asoc,
                       struct sctp_sndrcvinfo *sinfo, int msg_len);
void sctp_generate_fwdtsn(struct sctp_outq *q, __u32 sack_ctsn);
/* Uncork and flush an outqueue.  */
static inline void sctp_outq_cork(struct sctp_outq *q)
{
        q->cork = 1;
}

/* SCTP skb control block.
 * sctp_input_cb is currently used on rx and sock rx queue
 */
struct sctp_input_cb {
        union {
                struct inet_skb_parm    h4;
#if IS_ENABLED(CONFIG_IPV6)
                struct inet6_skb_parm   h6;
#endif
        } header;
        struct sctp_chunk *chunk;
        struct sctp_af *af;
        __be16 encap_port;
};
#define SCTP_INPUT_CB(__skb)        ((struct sctp_input_cb *)&((__skb)->cb[0]))

struct sctp_output_cb {
        struct sk_buff *last;
};
#define SCTP_OUTPUT_CB(__skb)        ((struct sctp_output_cb *)&((__skb)->cb[0]))

static inline const struct sk_buff *sctp_gso_headskb(const struct sk_buff *skb)
{
        const struct sctp_chunk *chunk = SCTP_INPUT_CB(skb)->chunk;

        return chunk->head_skb ? : skb;
}

/* These bind address data fields common between endpoints and associations */
struct sctp_bind_addr {

        /* RFC 2960 12.1 Parameters necessary for the SCTP instance
         *
         * SCTP Port:        The local SCTP port number the endpoint is
         *                bound to.
         */
        __u16 port;

        /* RFC 2960 12.1 Parameters necessary for the SCTP instance
         *
         * Address List: The list of IP addresses that this instance
         *        has bound.  This information is passed to one's
         *        peer(s) in INIT and INIT ACK chunks.
         */
        struct list_head address_list;
};

void sctp_bind_addr_init(struct sctp_bind_addr *, __u16 port);
void sctp_bind_addr_free(struct sctp_bind_addr *);
int sctp_bind_addr_copy(struct net *net, struct sctp_bind_addr *dest,
                        const struct sctp_bind_addr *src,
                        enum sctp_scope scope, gfp_t gfp,
                        int flags);
int sctp_bind_addr_dup(struct sctp_bind_addr *dest,
                        const struct sctp_bind_addr *src,
                        gfp_t gfp);
int sctp_add_bind_addr(struct sctp_bind_addr *, union sctp_addr *,
                       int new_size, __u8 addr_state, gfp_t gfp);
int sctp_del_bind_addr(struct sctp_bind_addr *, union sctp_addr *);
int sctp_bind_addr_match(struct sctp_bind_addr *, const union sctp_addr *,
                         struct sctp_sock *);
int sctp_bind_addr_conflict(struct sctp_bind_addr *, const union sctp_addr *,
                         struct sctp_sock *, struct sctp_sock *);
int sctp_bind_addr_state(const struct sctp_bind_addr *bp,
                         const union sctp_addr *addr);
int sctp_bind_addrs_check(struct sctp_sock *sp,
                          struct sctp_sock *sp2, int cnt2);
union sctp_addr *sctp_find_unmatch_addr(struct sctp_bind_addr        *bp,
                                        const union sctp_addr        *addrs,
                                        int                        addrcnt,
                                        struct sctp_sock        *opt);
union sctp_params sctp_bind_addrs_to_raw(const struct sctp_bind_addr *bp,
                                         int *addrs_len,
                                         gfp_t gfp);
int sctp_raw_to_bind_addrs(struct sctp_bind_addr *bp, __u8 *raw, int len,
                           __u16 port, gfp_t gfp);

enum sctp_scope sctp_scope(const union sctp_addr *addr);
int sctp_in_scope(struct net *net, const union sctp_addr *addr,
                  const enum sctp_scope scope);
int sctp_is_any(struct sock *sk, const union sctp_addr *addr);
int sctp_is_ep_boundall(struct sock *sk);


/* What type of endpoint?  */
enum sctp_endpoint_type {
        SCTP_EP_TYPE_SOCKET,
        SCTP_EP_TYPE_ASSOCIATION,
};

/*
 * A common base class to bridge the implementation view of a
 * socket (usually listening) endpoint versus an association's
 * local endpoint.
 * This common structure is useful for several purposes:
 *   1) Common interface for lookup routines.
 *        a) Subfunctions work for either endpoint or association
 *        b) Single interface to lookup allows hiding the lookup lock rather
 *           than acquiring it externally.
 *   2) Common interface for the inbound chunk handling/state machine.
 *   3) Common object handling routines for reference counting, etc.
 *   4) Disentangle association lookup from endpoint lookup, where we
 *        do not have to find our endpoint to find our association.
 *
 */

struct sctp_ep_common {
        /* Runtime type information.  What kind of endpoint is this? */
        enum sctp_endpoint_type type;

        /* Some fields to help us manage this object.
         *   refcnt   - Reference count access to this object.
         *   dead     - Do not attempt to use this object.
         */
        refcount_t    refcnt;
        bool            dead;

        /* What socket does this endpoint belong to?  */
        struct sock *sk;

        /* Cache netns and it won't change once set */
        struct net *net;

        /* This is where we receive inbound chunks.  */
        struct sctp_inq          inqueue;

        /* This substructure includes the defining parameters of the
         * endpoint:
         * bind_addr.port is our shared port number.
         * bind_addr.address_list is our set of local IP addresses.
         */
        struct sctp_bind_addr bind_addr;
};


/* RFC Section 1.4 Key Terms
 *
 * o SCTP endpoint: The logical sender/receiver of SCTP packets. On a
 *   multi-homed host, an SCTP endpoint is represented to its peers as a
 *   combination of a set of eligible destination transport addresses to
 *   which SCTP packets can be sent and a set of eligible source
 *   transport addresses from which SCTP packets can be received.
 *   All transport addresses used by an SCTP endpoint must use the
 *   same port number, but can use multiple IP addresses. A transport
 *   address used by an SCTP endpoint must not be used by another
 *   SCTP endpoint. In other words, a transport address is unique
 *   to an SCTP endpoint.
 *
 * From an implementation perspective, each socket has one of these.
 * A TCP-style socket will have exactly one association on one of
 * these.  An UDP-style socket will have multiple associations hanging
 * off one of these.
 */

struct sctp_endpoint {
        /* Common substructure for endpoint and association. */
        struct sctp_ep_common base;

        /* Fields to help us manage our entries in the hash tables. */
        struct hlist_node node;
        int hashent;

        /* Associations: A list of current associations and mappings
         *              to the data consumers for each association. This
         *              may be in the form of a hash table or other
         *              implementation dependent structure. The data
         *              consumers may be process identification
         *              information such as file descriptors, named pipe
         *              pointer, or table pointers dependent on how SCTP
         *              is implemented.
         */
        /* This is really a list of struct sctp_association entries. */
        struct list_head asocs;

        /* Cookie authentication key used by this endpoint */
        struct hmac_sha256_key cookie_auth_key;

        /* sendbuf acct. policy.        */
        __u32 sndbuf_policy;

        /* rcvbuf acct. policy.        */
        __u32 rcvbuf_policy;

        /* SCTP-AUTH: hmacs for the endpoint encoded into parameter */
         struct sctp_hmac_algo_param *auth_hmacs_list;

        /* SCTP-AUTH: chunks to authenticate encoded into parameter */
        struct sctp_chunks_param *auth_chunk_list;

        /* SCTP-AUTH: endpoint shared keys */
        struct list_head endpoint_shared_keys;
        __u16 active_key_id;
        __u8  ecn_enable:1,
              auth_enable:1,
              intl_enable:1,
              prsctp_enable:1,
              asconf_enable:1,
              reconf_enable:1;

        __u8  strreset_enable;
        struct rcu_head rcu;
};

/* Recover the outer endpoint structure. */
static inline struct sctp_endpoint *sctp_ep(struct sctp_ep_common *base)
{
        struct sctp_endpoint *ep;

        ep = container_of(base, struct sctp_endpoint, base);
        return ep;
}

/* These are function signatures for manipulating endpoints.  */
struct sctp_endpoint *sctp_endpoint_new(struct sock *, gfp_t);
void sctp_endpoint_free(struct sctp_endpoint *);
void sctp_endpoint_put(struct sctp_endpoint *);
int sctp_endpoint_hold(struct sctp_endpoint *ep);
void sctp_endpoint_add_asoc(struct sctp_endpoint *, struct sctp_association *);
struct sctp_association *sctp_endpoint_lookup_assoc(
        const struct sctp_endpoint *ep,
        const union sctp_addr *paddr,
        struct sctp_transport **);
bool sctp_endpoint_is_peeled_off(struct sctp_endpoint *ep,
                                 const union sctp_addr *paddr);
struct sctp_endpoint *sctp_endpoint_is_match(struct sctp_endpoint *ep,
                                             struct net *net,
                                             const union sctp_addr *laddr,
                                             int dif, int sdif);
bool sctp_has_association(struct net *net, const union sctp_addr *laddr,
                          const union sctp_addr *paddr, int dif, int sdif);

int sctp_verify_init(struct net *net, const struct sctp_endpoint *ep,
                     const struct sctp_association *asoc,
                     enum sctp_cid cid, struct sctp_init_chunk *peer_init,
                     struct sctp_chunk *chunk, struct sctp_chunk **err_chunk);
int sctp_process_init(struct sctp_association *, struct sctp_chunk *chunk,
                      const union sctp_addr *peer,
                      struct sctp_init_chunk *init, gfp_t gfp);
__u32 sctp_generate_tag(const struct sctp_endpoint *);
__u32 sctp_generate_tsn(const struct sctp_endpoint *);

struct sctp_inithdr_host {
        __u32 init_tag;
        __u32 a_rwnd;
        __u16 num_outbound_streams;
        __u16 num_inbound_streams;
        __u32 initial_tsn;
};

struct sctp_stream_priorities {
        /* List of priorities scheduled */
        struct list_head prio_sched;
        /* List of streams scheduled */
        struct list_head active;
        /* The next stream in line */
        struct sctp_stream_out_ext *next;
        __u16 prio;
        __u16 users;
};

struct sctp_stream_out_ext {
        __u64 abandoned_unsent[SCTP_PR_INDEX(MAX) + 1];
        __u64 abandoned_sent[SCTP_PR_INDEX(MAX) + 1];
        struct list_head outq; /* chunks enqueued by this stream */
        union {
                struct {
                        /* Scheduled streams list */
                        struct list_head prio_list;
                        struct sctp_stream_priorities *prio_head;
                };
                /* Fields used by RR scheduler */
                struct {
                        struct list_head rr_list;
                };
                struct {
                        struct list_head fc_list;
                        __u32 fc_length;
                        __u16 fc_weight;
                };
        };
};

struct sctp_stream_out {
        union {
                __u32 mid;
                __u16 ssn;
        };
        __u32 mid_uo;
        struct sctp_stream_out_ext *ext;
        __u8 state;
};

struct sctp_stream_in {
        union {
                __u32 mid;
                __u16 ssn;
        };
        __u32 mid_uo;
        __u32 fsn;
        __u32 fsn_uo;
        char pd_mode;
        char pd_mode_uo;
};

struct sctp_stream {
        GENRADIX(struct sctp_stream_out) out;
        GENRADIX(struct sctp_stream_in)        in;

        __u16 outcnt;
        __u16 incnt;
        /* Current stream being sent, if any */
        struct sctp_stream_out *out_curr;
        union {
                /* Fields used by priority scheduler */
                struct {
                        /* List of priorities scheduled */
                        struct list_head prio_list;
                };
                /* Fields used by RR scheduler */
                struct {
                        /* List of streams scheduled */
                        struct list_head rr_list;
                        /* The next stream in line */
                        struct sctp_stream_out_ext *rr_next;
                };
                struct {
                        struct list_head fc_list;
                };
        };
        struct sctp_stream_interleave *si;
};

static inline struct sctp_stream_out *sctp_stream_out(
        struct sctp_stream *stream,
        __u16 sid)
{
        return genradix_ptr(&stream->out, sid);
}

static inline struct sctp_stream_in *sctp_stream_in(
        struct sctp_stream *stream,
        __u16 sid)
{
        return genradix_ptr(&stream->in, sid);
}

#define SCTP_SO(s, i) sctp_stream_out((s), (i))
#define SCTP_SI(s, i) sctp_stream_in((s), (i))

#define SCTP_STREAM_CLOSED                0x00
#define SCTP_STREAM_OPEN                0x01

static inline __u16 sctp_datachk_len(const struct sctp_stream *stream)
{
        return stream->si->data_chunk_len;
}

static inline __u16 sctp_datahdr_len(const struct sctp_stream *stream)
{
        return stream->si->data_chunk_len - sizeof(struct sctp_chunkhdr);
}

static inline __u16 sctp_ftsnchk_len(const struct sctp_stream *stream)
{
        return stream->si->ftsn_chunk_len;
}

static inline __u16 sctp_ftsnhdr_len(const struct sctp_stream *stream)
{
        return stream->si->ftsn_chunk_len - sizeof(struct sctp_chunkhdr);
}

/* SCTP_GET_ASSOC_STATS counters */
struct sctp_priv_assoc_stats {
        /* Maximum observed rto in the association during subsequent
         * observations. Value is set to 0 if no RTO measurement took place
         * The transport where the max_rto was observed is returned in
         * obs_rto_ipaddr
         */
        struct sockaddr_storage obs_rto_ipaddr;
        __u64 max_obs_rto;
        /* Total In and Out SACKs received and sent */
        __u64 isacks;
        __u64 osacks;
        /* Total In and Out packets received and sent */
        __u64 opackets;
        __u64 ipackets;
        /* Total retransmitted chunks */
        __u64 rtxchunks;
        /* TSN received > next expected */
        __u64 outofseqtsns;
        /* Duplicate Chunks received */
        __u64 idupchunks;
        /* Gap Ack Blocks received */
        __u64 gapcnt;
        /* Unordered data chunks sent and received */
        __u64 ouodchunks;
        __u64 iuodchunks;
        /* Ordered data chunks sent and received */
        __u64 oodchunks;
        __u64 iodchunks;
        /* Control chunks sent and received */
        __u64 octrlchunks;
        __u64 ictrlchunks;
};

/* RFC2960
 *
 * 12. Recommended Transmission Control Block (TCB) Parameters
 *
 * This section details a recommended set of parameters that should
 * be contained within the TCB for an implementation. This section is
 * for illustrative purposes and should not be deemed as requirements
 * on an implementation or as an exhaustive list of all parameters
 * inside an SCTP TCB. Each implementation may need its own additional
 * parameters for optimization.
 */


/* Here we have information about each individual association. */
struct sctp_association {

        /* A base structure common to endpoint and association.
         * In this context, it represents the associations's view
         * of the local endpoint of the association.
         */
        struct sctp_ep_common base;

        /* Associations on the same socket. */
        struct list_head asocs;

        /* association id. */
        sctp_assoc_t assoc_id;

        /* This is our parent endpoint.         */
        struct sctp_endpoint *ep;

        /* These are those association elements needed in the cookie.  */
        struct sctp_cookie c;

        /* This is all information about our peer.  */
        struct {
                /* transport_addr_list
                 *
                 * Peer               : A list of SCTP transport addresses that the
                 * Transport   : peer is bound to. This information is derived
                 * Address     : from the INIT or INIT ACK and is used to
                 * List               : associate an inbound packet with a given
                 *               : association. Normally this information is
                 *               : hashed or keyed for quick lookup and access
                 *               : of the TCB.
                 *               : The list is also initialized with the list
                 *               : of addresses passed with the sctp_connectx()
                 *               : call.
                 *
                 * It is a list of SCTP_transport's.
                 */
                struct list_head transport_addr_list;

                /* rwnd
                 *
                 * Peer Rwnd   : Current calculated value of the peer's rwnd.
                 */
                __u32 rwnd;

                /* transport_count
                 *
                 * Peer        : A count of the number of peer addresses
                 * Transport   : in the Peer Transport Address List.
                 * Address     :
                 * Count       :
                 */
                __u16 transport_count;

                /* port
                 *   The transport layer port number.
                 */
                __u16 port;

                /* primary_path
                 *
                 * Primary     : This is the current primary destination
                 * Path               : transport address of the peer endpoint.  It
                 *               : may also specify a source transport address
                 *               : on this endpoint.
                 *
                 * All of these paths live on transport_addr_list.
                 *
                 * At the bakeoffs, we discovered that the intent of
                 * primaryPath is that it only changes when the ULP
                 * asks to have it changed.  We add the activePath to
                 * designate the connection we are currently using to
                 * transmit new data and most control chunks.
                 */
                struct sctp_transport *primary_path;

                /* Cache the primary path address here, when we
                 * need a an address for msg_name.
                 */
                union sctp_addr primary_addr;

                /* active_path
                 *   The path that we are currently using to
                 *   transmit new data and most control chunks.
                 */
                struct sctp_transport *active_path;

                /* retran_path
                 *
                 * RFC2960 6.4 Multi-homed SCTP Endpoints
                 * ...
                 * Furthermore, when its peer is multi-homed, an
                 * endpoint SHOULD try to retransmit a chunk to an
                 * active destination transport address that is
                 * different from the last destination address to
                 * which the DATA chunk was sent.
                 */
                struct sctp_transport *retran_path;

                /* Pointer to last transport I have sent on.  */
                struct sctp_transport *last_sent_to;

                /* This is the last transport I have received DATA on.        */
                struct sctp_transport *last_data_from;

                /*
                 * Mapping  An array of bits or bytes indicating which out of
                 * Array    order TSN's have been received (relative to the
                 *            Last Rcvd TSN). If no gaps exist, i.e. no out of
                 *            order packets have been received, this array
                 *            will be set to all zero. This structure may be
                 *            in the form of a circular buffer or bit array.
                 *
                 * Last Rcvd   : This is the last TSN received in
                 * TSN               : sequence. This value is set initially by
                 *               : taking the peer's Initial TSN, received in
                 *               : the INIT or INIT ACK chunk, and subtracting
                 *               : one from it.
                 *
                 * Throughout most of the specification this is called the
                 * "Cumulative TSN ACK Point".        In this case, we
                 * ignore the advice in 12.2 in favour of the term
                 * used in the bulk of the text.  This value is hidden
                 * in tsn_map--we get it by calling sctp_tsnmap_get_ctsn().
                 */
                struct sctp_tsnmap tsn_map;

                /* This mask is used to disable sending the ASCONF chunk
                 * with specified parameter to peer.
                 */
                __be16 addip_disabled_mask;

                /* These are capabilities which our peer advertised.  */
                __u16        ecn_capable:1,      /* Can peer do ECN? */
                        ipv4_address:1,     /* Peer understands IPv4 addresses? */
                        ipv6_address:1,     /* Peer understands IPv6 addresses? */
                        asconf_capable:1,   /* Does peer support ADDIP? */
                        prsctp_capable:1,   /* Can peer do PR-SCTP? */
                        reconf_capable:1,   /* Can peer do RE-CONFIG? */
                        intl_capable:1,     /* Can peer do INTERLEAVE */
                        auth_capable:1,     /* Is peer doing SCTP-AUTH? */
                        /* sack_needed:
                         *   This flag indicates if the next received
                         *   packet is to be responded to with a
                         *   SACK. This is initialized to 0.  When a packet
                         *   is received sack_cnt is incremented. If this value
                         *   reaches 2 or more, a SACK is sent and the
                         *   value is reset to 0. Note: This is used only
                         *   when no DATA chunks are received out of
                         *   order.  When DATA chunks are out of order,
                         *   SACK's are not delayed (see Section 6).
                         */
                        sack_needed:1,     /* Do we need to sack the peer? */
                        sack_generation:1,
                        zero_window_announced:1;

                __u32        sack_cnt;

                __u32   adaptation_ind;         /* Adaptation Code point. */

                struct sctp_inithdr_host i;
                void *cookie;
                int cookie_len;

                /* ADDIP Section 4.2 Upon reception of an ASCONF Chunk.
                 * C1) ... "Peer-Serial-Number'. This value MUST be initialized to the
                 * Initial TSN Value minus 1
                 */
                __u32 addip_serial;

                /* SCTP-AUTH: We need to know pears random number, hmac list
                 * and authenticated chunk list.  All that is part of the
                 * cookie and these are just pointers to those locations
                 */
                struct sctp_random_param *peer_random;
                struct sctp_chunks_param *peer_chunks;
                struct sctp_hmac_algo_param *peer_hmacs;
        } peer;

        /* State       : A state variable indicating what state the
         *               : association is in, i.e. COOKIE-WAIT,
         *               : COOKIE-ECHOED, ESTABLISHED, SHUTDOWN-PENDING,
         *               : SHUTDOWN-SENT, SHUTDOWN-RECEIVED, SHUTDOWN-ACK-SENT.
         *
         *                Note: No "CLOSED" state is illustrated since if a
         *                association is "CLOSED" its TCB SHOULD be removed.
         *
         *                In this implementation we DO have a CLOSED
         *                state which is used during initiation and shutdown.
         *
         *                State takes values from SCTP_STATE_*.
         */
        enum sctp_state state;

        /* Overall     : The overall association error count.
         * Error Count : [Clear this any time I get something.]
         */
        int overall_error_count;

        /* The cookie life I award for any cookie.  */
        ktime_t cookie_life;

        /* These are the association's initial, max, and min RTO values.
         * These values will be initialized by system defaults, but can
         * be modified via the SCTP_RTOINFO socket option.
         */
        unsigned long rto_initial;
        unsigned long rto_max;
        unsigned long rto_min;

        /* Maximum number of new data packets that can be sent in a burst.  */
        int max_burst;

        /* This is the max_retrans value for the association.  This value will
         * be initialized from system defaults, but can be
         * modified by the SCTP_ASSOCINFO socket option.
         */
        int max_retrans;

        /* This is the partially failed retrans value for the transport
         * and will be initialized from the assocs value.  This can be
         * changed using the SCTP_PEER_ADDR_THLDS socket option
         */
        __u16 pf_retrans;
        /* Used for primary path switchover. */
        __u16 ps_retrans;

        /* Maximum number of times the endpoint will retransmit INIT  */
        __u16 max_init_attempts;

        /* How many times have we resent an INIT? */
        __u16 init_retries;

        /* The largest timeout or RTO value to use in attempting an INIT */
        unsigned long max_init_timeo;

        /* Heartbeat interval: The endpoint sends out a Heartbeat chunk to
         * the destination address every heartbeat interval. This value
         * will be inherited by all new transports.
         */
        unsigned long hbinterval;
        unsigned long probe_interval;

        __be16 encap_port;

        /* This is the max_retrans value for new transports in the
         * association.
         */
        __u16 pathmaxrxt;

        __u32 flowlabel;
        __u8  dscp;

        /* Flag that path mtu update is pending */
        __u8   pmtu_pending;

        /* Association : The smallest PMTU discovered for all of the
         * PMTU               : peer's transport addresses.
         */
        __u32 pathmtu;

        /* Flags controlling Heartbeat, SACK delay, and Path MTU Discovery. */
        __u32 param_flags;

        __u32 sackfreq;
        /* SACK delay timeout */
        unsigned long sackdelay;

        unsigned long timeouts[SCTP_NUM_TIMEOUT_TYPES];
        struct timer_list timers[SCTP_NUM_TIMEOUT_TYPES];

        /* Transport to which SHUTDOWN chunk was last sent.  */
        struct sctp_transport *shutdown_last_sent_to;

        /* Transport to which INIT chunk was last sent.  */
        struct sctp_transport *init_last_sent_to;

        /* How many times have we resent a SHUTDOWN */
        int shutdown_retries;

        /* Next TSN    : The next TSN number to be assigned to a new
         *               : DATA chunk.  This is sent in the INIT or INIT
         *               : ACK chunk to the peer and incremented each
         *               : time a DATA chunk is assigned a TSN
         *               : (normally just prior to transmit or during
         *               : fragmentation).
         */
        __u32 next_tsn;

        /*
         * Last Rcvd   : This is the last TSN received in sequence.  This value
         * TSN               : is set initially by taking the peer's Initial TSN,
         *               : received in the INIT or INIT ACK chunk, and
         *               : subtracting one from it.
         *
         * Most of RFC 2960 refers to this as the Cumulative TSN Ack Point.
         */

        __u32 ctsn_ack_point;

        /* PR-SCTP Advanced.Peer.Ack.Point */
        __u32 adv_peer_ack_point;

        /* Highest TSN that is acknowledged by incoming SACKs. */
        __u32 highest_sacked;

        /* TSN marking the fast recovery exit point */
        __u32 fast_recovery_exit;

        /* Flag to track the current fast recovery state */
        __u8 fast_recovery;

        /* The number of unacknowledged data chunks.  Reported through
         * the SCTP_STATUS sockopt.
         */
        __u16 unack_data;

        /* The total number of data chunks that we've had to retransmit
         * as the result of a T3 timer expiration
         */
        __u32 rtx_data_chunks;

        /* This is the association's receive buffer space.  This value is used
         * to set a_rwnd field in an INIT or a SACK chunk.
         */
        __u32 rwnd;

        /* This is the last advertised value of rwnd over a SACK chunk. */
        __u32 a_rwnd;

        /* Number of bytes by which the rwnd has slopped.  The rwnd is allowed
         * to slop over a maximum of the association's frag_point.
         */
        __u32 rwnd_over;

        /* Keeps treack of rwnd pressure.  This happens when we have
         * a window, but not receive buffer (i.e small packets).  This one
         * is releases slowly (1 PMTU at a time ).
         */
        __u32 rwnd_press;

        /* This is the sndbuf size in use for the association.
         * This corresponds to the sndbuf size for the association,
         * as specified in the sk->sndbuf.
         */
        int sndbuf_used;

        /* This is the amount of memory that this association has allocated
         * in the receive path at any given time.
         */
        atomic_t rmem_alloc;

        /* This is the wait queue head for send requests waiting on
         * the association sndbuf space.
         */
        wait_queue_head_t        wait;

        /* The message size at which SCTP fragmentation will occur. */
        __u32 frag_point;
        __u32 user_frag;

        /* Counter used to count INIT errors. */
        int init_err_counter;

        /* Count the number of INIT cycles (for doubling timeout). */
        int init_cycle;

        /* Default send parameters. */
        __u16 default_stream;
        __u16 default_flags;
        __u32 default_ppid;
        __u32 default_context;
        __u32 default_timetolive;

        /* Default receive parameters */
        __u32 default_rcv_context;

        /* Stream arrays */
        struct sctp_stream stream;

        /* All outbound chunks go through this structure.  */
        struct sctp_outq outqueue;

        /* A smart pipe that will handle reordering and fragmentation,
         * as well as handle passing events up to the ULP.
         */
        struct sctp_ulpq ulpq;

        /* Last TSN that caused an ECNE Chunk to be sent.  */
        __u32 last_ecne_tsn;

        /* Last TSN that caused a CWR Chunk to be sent.         */
        __u32 last_cwr_tsn;

        /* How many duplicated TSNs have we seen?  */
        int numduptsns;

        /* These are to support
         * "SCTP Extensions for Dynamic Reconfiguration of IP Addresses
         *  and Enforcement of Flow and Message Limits"
         * <draft-ietf-tsvwg-addip-sctp-02.txt>
         * or "ADDIP" for short.
         */



        /* ADDIP Section 4.1.1 Congestion Control of ASCONF Chunks
         *
         * R1) One and only one ASCONF Chunk MAY be in transit and
         * unacknowledged at any one time.  If a sender, after sending
         * an ASCONF chunk, decides it needs to transfer another
         * ASCONF Chunk, it MUST wait until the ASCONF-ACK Chunk
         * returns from the previous ASCONF Chunk before sending a
         * subsequent ASCONF. Note this restriction binds each side,
         * so at any time two ASCONF may be in-transit on any given
         * association (one sent from each endpoint).
         *
         * [This is our one-and-only-one ASCONF in flight.  If we do
         * not have an ASCONF in flight, this is NULL.]
         */
        struct sctp_chunk *addip_last_asconf;

        /* ADDIP Section 5.2 Upon reception of an ASCONF Chunk.
         *
         * This is needed to implement items E1 - E4 of the updated
         * spec.  Here is the justification:
         *
         * Since the peer may bundle multiple ASCONF chunks toward us,
         * we now need the ability to cache multiple ACKs.  The section
         * describes in detail how they are cached and cleaned up.
         */
        struct list_head asconf_ack_list;

        /* These ASCONF chunks are waiting to be sent.
         *
         * These chunks can't be pushed to outqueue until receiving
         * ASCONF_ACK for the previous ASCONF indicated by
         * addip_last_asconf, so as to guarantee that only one ASCONF
         * is in flight at any time.
         *
         * ADDIP Section 4.1.1 Congestion Control of ASCONF Chunks
         *
         * In defining the ASCONF Chunk transfer procedures, it is
         * essential that these transfers MUST NOT cause congestion
         * within the network.        To achieve this, we place these
         * restrictions on the transfer of ASCONF Chunks:
         *
         * R1) One and only one ASCONF Chunk MAY be in transit and
         * unacknowledged at any one time.  If a sender, after sending
         * an ASCONF chunk, decides it needs to transfer another
         * ASCONF Chunk, it MUST wait until the ASCONF-ACK Chunk
         * returns from the previous ASCONF Chunk before sending a
         * subsequent ASCONF. Note this restriction binds each side,
         * so at any time two ASCONF may be in-transit on any given
         * association (one sent from each endpoint).
         *
         *
         * [I really think this is EXACTLY the sort of intelligence
         *  which already resides in sctp_outq.         Please move this
         *  queue and its supporting logic down there.        --piggy]
         */
        struct list_head addip_chunk_list;

        /* ADDIP Section 4.1 ASCONF Chunk Procedures
         *
         * A2) A serial number should be assigned to the Chunk. The
         * serial number SHOULD be a monotonically increasing
         * number. The serial number SHOULD be initialized at
         * the start of the association to the same value as the
         * Initial TSN and every time a new ASCONF chunk is created
         * it is incremented by one after assigning the serial number
         * to the newly created chunk.
         *
         * ADDIP
         * 3.1.1  Address/Stream Configuration Change Chunk (ASCONF)
         *
         * Serial Number : 32 bits (unsigned integer)
         *
         * This value represents a Serial Number for the ASCONF
         * Chunk. The valid range of Serial Number is from 0 to
         * 4294967295 (2^32 - 1).  Serial Numbers wrap back to 0
         * after reaching 4294967295.
         */
        __u32 addip_serial;
        int src_out_of_asoc_ok;
        union sctp_addr *asconf_addr_del_pending;
        struct sctp_transport *new_transport;

        /* SCTP AUTH: list of the endpoint shared keys.  These
         * keys are provided out of band by the user application
         * and can't change during the lifetime of the association
         */
        struct list_head endpoint_shared_keys;

        /* SCTP AUTH:
         * The current generated association shared key (secret)
         */
        struct sctp_auth_bytes *asoc_shared_key;
        struct sctp_shared_key *shkey;

        /* SCTP AUTH: hmac id of the first peer requested algorithm
         * that we support.
         */
        __u16 default_hmac_id;

        __u16 active_key_id;

        __u8 need_ecne:1,        /* Need to send an ECNE Chunk? */
             temp:1,                /* Is it a temporary association? */
             pf_expose:2,       /* Expose pf state? */
             force_delay:1;

        __u8 strreset_enable;
        __u8 strreset_outstanding; /* request param count on the fly */

        __u32 strreset_outseq; /* Update after receiving response */
        __u32 strreset_inseq; /* Update after receiving request */
        __u32 strreset_result[2]; /* save the results of last 2 responses */

        struct sctp_chunk *strreset_chunk; /* save request chunk */

        struct sctp_priv_assoc_stats stats;

        int sent_cnt_removable;

        __u16 subscribe;

        __u64 abandoned_unsent[SCTP_PR_INDEX(MAX) + 1];
        __u64 abandoned_sent[SCTP_PR_INDEX(MAX) + 1];

        /* Security identifiers from incoming (INIT). These are set by
         * security_sctp_assoc_request(). These will only be used by
         * SCTP TCP type sockets and peeled off connections as they
         * cause a new socket to be generated. security_sctp_sk_clone()
         * will then plug these into the new socket.
         */

        u32 secid;
        u32 peer_secid;

        struct rcu_head rcu;
};


/* An eyecatcher for determining if we are really looking at an
 * association data structure.
 */
enum {
        SCTP_ASSOC_EYECATCHER = 0xa550c123,
};

/* Recover the outer association structure. */
static inline struct sctp_association *sctp_assoc(struct sctp_ep_common *base)
{
        struct sctp_association *asoc;

        asoc = container_of(base, struct sctp_association, base);
        return asoc;
}

/* These are function signatures for manipulating associations.         */


struct sctp_association *
sctp_association_new(const struct sctp_endpoint *ep, const struct sock *sk,
                     enum sctp_scope scope, gfp_t gfp);
void sctp_association_free(struct sctp_association *);
void sctp_association_put(struct sctp_association *);
void sctp_association_hold(struct sctp_association *);

struct sctp_transport *sctp_assoc_choose_alter_transport(
        struct sctp_association *, struct sctp_transport *);
void sctp_assoc_update_retran_path(struct sctp_association *);
struct sctp_transport *sctp_assoc_lookup_paddr(const struct sctp_association *,
                                          const union sctp_addr *);
int sctp_assoc_lookup_laddr(struct sctp_association *asoc,
                            const union sctp_addr *laddr);
struct sctp_transport *sctp_assoc_add_peer(struct sctp_association *,
                                     const union sctp_addr *address,
                                     const gfp_t gfp,
                                     const int peer_state);
void sctp_assoc_rm_peer(struct sctp_association *asoc,
                         struct sctp_transport *peer);
void sctp_assoc_control_transport(struct sctp_association *asoc,
                                  struct sctp_transport *transport,
                                  enum sctp_transport_cmd command,
                                  sctp_sn_error_t error);
struct sctp_transport *sctp_assoc_lookup_tsn(struct sctp_association *, __u32);
void sctp_assoc_migrate(struct sctp_association *, struct sock *);
int sctp_assoc_update(struct sctp_association *old,
                      struct sctp_association *new);

__u32 sctp_association_get_next_tsn(struct sctp_association *);

void sctp_assoc_update_frag_point(struct sctp_association *asoc);
void sctp_assoc_set_pmtu(struct sctp_association *asoc, __u32 pmtu);
void sctp_assoc_sync_pmtu(struct sctp_association *asoc);
void sctp_assoc_rwnd_increase(struct sctp_association *, unsigned int);
void sctp_assoc_rwnd_decrease(struct sctp_association *, unsigned int);
void sctp_assoc_set_primary(struct sctp_association *,
                            struct sctp_transport *);
void sctp_assoc_del_nonprimary_peers(struct sctp_association *,
                                    struct sctp_transport *);
int sctp_assoc_set_bind_addr_from_ep(struct sctp_association *asoc,
                                     enum sctp_scope scope, gfp_t gfp);
int sctp_assoc_set_bind_addr_from_cookie(struct sctp_association *,
                                         struct sctp_cookie*,
                                         gfp_t gfp);
int sctp_assoc_set_id(struct sctp_association *, gfp_t);
void sctp_assoc_clean_asconf_ack_cache(const struct sctp_association *asoc);
struct sctp_chunk *sctp_assoc_lookup_asconf_ack(
                                        const struct sctp_association *asoc,
                                        __be32 serial);
void sctp_asconf_queue_teardown(struct sctp_association *asoc);

int sctp_cmp_addr_exact(const union sctp_addr *ss1,
                        const union sctp_addr *ss2);
struct sctp_chunk *sctp_get_ecne_prepend(struct sctp_association *asoc);

/* A convenience structure to parse out SCTP specific CMSGs. */
struct sctp_cmsgs {
        struct sctp_initmsg *init;
        struct sctp_sndrcvinfo *srinfo;
        struct sctp_sndinfo *sinfo;
        struct sctp_prinfo *prinfo;
        struct sctp_authinfo *authinfo;
        struct msghdr *addrs_msg;
};

/* Structure for tracking memory objects */
struct sctp_dbg_objcnt_entry {
        char *label;
        atomic_t *counter;
};

#endif /* __sctp_structs_h__ */







































































































    6 
































































































































































































































































































































































































































































































































    6 



    6 



























































































































    6 









    6 


















  104 





    1 




    1 

    1 




    1 


  103 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
// SPDX-License-Identifier: GPL-2.0-only
/*
 * xfrm_replay.c - xfrm replay detection, derived from xfrm_state.c.
 *
 * Copyright (C) 2010 secunet Security Networks AG
 * Copyright (C) 2010 Steffen Klassert <steffen.klassert@secunet.com>
 */

#include <linux/export.h>
#include <net/xfrm.h>

u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq)
{
        u32 seq, seq_hi, bottom;
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;

        if (!(x->props.flags & XFRM_STATE_ESN))
                return 0;

        seq = ntohl(net_seq);
        seq_hi = replay_esn->seq_hi;
        bottom = replay_esn->seq - replay_esn->replay_window + 1;

        if (likely(replay_esn->seq >= replay_esn->replay_window - 1)) {
                /* A. same subspace */
                if (unlikely(seq < bottom))
                        seq_hi++;
        } else {
                /* B. window spans two subspaces */
                if (unlikely(seq >= bottom))
                        seq_hi--;
        }

        return seq_hi;
}
EXPORT_SYMBOL(xfrm_replay_seqhi);

static void xfrm_replay_notify_bmp(struct xfrm_state *x, int event);
static void xfrm_replay_notify_esn(struct xfrm_state *x, int event);

void xfrm_replay_notify(struct xfrm_state *x, int event)
{
        struct km_event c;
        /* we send notify messages in case
         *  1. we updated on of the sequence numbers, and the seqno difference
         *     is at least x->replay_maxdiff, in this case we also update the
         *     timeout of our timer function
         *  2. if x->replay_maxage has elapsed since last update,
         *     and there were changes
         *
         *  The state structure must be locked!
         */

        switch (x->repl_mode) {
        case XFRM_REPLAY_MODE_LEGACY:
                break;
        case XFRM_REPLAY_MODE_BMP:
                xfrm_replay_notify_bmp(x, event);
                return;
        case XFRM_REPLAY_MODE_ESN:
                xfrm_replay_notify_esn(x, event);
                return;
        }

        switch (event) {
        case XFRM_REPLAY_UPDATE:
                if (!x->replay_maxdiff ||
                    ((x->replay.seq - x->preplay.seq < x->replay_maxdiff) &&
                    (x->replay.oseq - x->preplay.oseq < x->replay_maxdiff))) {
                        if (x->xflags & XFRM_TIME_DEFER)
                                event = XFRM_REPLAY_TIMEOUT;
                        else
                                return;
                }

                break;

        case XFRM_REPLAY_TIMEOUT:
                if (memcmp(&x->replay, &x->preplay,
                           sizeof(struct xfrm_replay_state)) == 0) {
                        x->xflags |= XFRM_TIME_DEFER;
                        return;
                }

                break;
        }

        memcpy(&x->preplay, &x->replay, sizeof(struct xfrm_replay_state));
        c.event = XFRM_MSG_NEWAE;
        c.data.aevent = event;
        km_state_notify(x, &c);

        if (x->replay_maxage &&
            !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
                x->xflags &= ~XFRM_TIME_DEFER;
}

static int __xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb)
{
        int err = 0;
        struct net *net = xs_net(x);

        if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
                XFRM_SKB_CB(skb)->seq.output.low = ++x->replay.oseq;
                XFRM_SKB_CB(skb)->seq.output.hi = 0;
                if (unlikely(x->replay.oseq == 0) &&
                    !(x->props.extra_flags & XFRM_SA_XFLAG_OSEQ_MAY_WRAP)) {
                        x->replay.oseq--;
                        xfrm_audit_state_replay_overflow(x, skb);
                        err = -EOVERFLOW;

                        return err;
                }
                if (xfrm_aevent_is_on(net))
                        xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
        }

        return err;
}

static int xfrm_replay_check_legacy(struct xfrm_state *x,
                                    struct sk_buff *skb, __be32 net_seq)
{
        u32 diff;
        u32 seq = ntohl(net_seq);

        if (!x->props.replay_window)
                return 0;

        if (unlikely(seq == 0))
                goto err;

        if (likely(seq > x->replay.seq))
                return 0;

        diff = x->replay.seq - seq;
        if (diff >= x->props.replay_window) {
                x->stats.replay_window++;
                goto err;
        }

        if (x->replay.bitmap & (1U << diff)) {
                x->stats.replay++;
                goto err;
        }
        return 0;

err:
        xfrm_audit_state_replay(x, skb, net_seq);
        return -EINVAL;
}

static void xfrm_replay_advance_bmp(struct xfrm_state *x, __be32 net_seq);
static void xfrm_replay_advance_esn(struct xfrm_state *x, __be32 net_seq);

void xfrm_replay_advance(struct xfrm_state *x, __be32 net_seq)
{
        u32 diff, seq;

        switch (x->repl_mode) {
        case XFRM_REPLAY_MODE_LEGACY:
                break;
        case XFRM_REPLAY_MODE_BMP:
                return xfrm_replay_advance_bmp(x, net_seq);
        case XFRM_REPLAY_MODE_ESN:
                return xfrm_replay_advance_esn(x, net_seq);
        }

        if (!x->props.replay_window)
                return;

        seq = ntohl(net_seq);
        if (seq > x->replay.seq) {
                diff = seq - x->replay.seq;
                if (diff < x->props.replay_window)
                        x->replay.bitmap = ((x->replay.bitmap) << diff) | 1;
                else
                        x->replay.bitmap = 1;
                x->replay.seq = seq;
        } else {
                diff = x->replay.seq - seq;
                x->replay.bitmap |= (1U << diff);
        }

        if (xfrm_aevent_is_on(xs_net(x)))
                xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
}

static int xfrm_replay_overflow_bmp(struct xfrm_state *x, struct sk_buff *skb)
{
        int err = 0;
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
        struct net *net = xs_net(x);

        if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
                XFRM_SKB_CB(skb)->seq.output.low = ++replay_esn->oseq;
                XFRM_SKB_CB(skb)->seq.output.hi = 0;
                if (unlikely(replay_esn->oseq == 0) &&
                    !(x->props.extra_flags & XFRM_SA_XFLAG_OSEQ_MAY_WRAP)) {
                        replay_esn->oseq--;
                        xfrm_audit_state_replay_overflow(x, skb);
                        err = -EOVERFLOW;

                        return err;
                }
                if (xfrm_aevent_is_on(net))
                        xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
        }

        return err;
}

static int xfrm_replay_check_bmp(struct xfrm_state *x,
                                 struct sk_buff *skb, __be32 net_seq)
{
        unsigned int bitnr, nr;
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
        u32 pos;
        u32 seq = ntohl(net_seq);
        u32 diff =  replay_esn->seq - seq;

        if (!replay_esn->replay_window)
                return 0;

        if (unlikely(seq == 0))
                goto err;

        if (likely(seq > replay_esn->seq))
                return 0;

        if (diff >= replay_esn->replay_window) {
                x->stats.replay_window++;
                goto err;
        }

        pos = (replay_esn->seq - 1) % replay_esn->replay_window;

        if (pos >= diff)
                bitnr = (pos - diff) % replay_esn->replay_window;
        else
                bitnr = replay_esn->replay_window - (diff - pos);

        nr = bitnr >> 5;
        bitnr = bitnr & 0x1F;
        if (replay_esn->bmp[nr] & (1U << bitnr))
                goto err_replay;

        return 0;

err_replay:
        x->stats.replay++;
err:
        xfrm_audit_state_replay(x, skb, net_seq);
        return -EINVAL;
}

static void xfrm_replay_advance_bmp(struct xfrm_state *x, __be32 net_seq)
{
        unsigned int bitnr, nr, i;
        u32 diff;
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
        u32 seq = ntohl(net_seq);
        u32 pos;

        if (!replay_esn->replay_window)
                return;

        pos = (replay_esn->seq - 1) % replay_esn->replay_window;

        if (seq > replay_esn->seq) {
                diff = seq - replay_esn->seq;

                if (diff < replay_esn->replay_window) {
                        for (i = 1; i < diff; i++) {
                                bitnr = (pos + i) % replay_esn->replay_window;
                                nr = bitnr >> 5;
                                bitnr = bitnr & 0x1F;
                                replay_esn->bmp[nr] &=  ~(1U << bitnr);
                        }
                } else {
                        nr = (replay_esn->replay_window - 1) >> 5;
                        for (i = 0; i <= nr; i++)
                                replay_esn->bmp[i] = 0;
                }

                bitnr = (pos + diff) % replay_esn->replay_window;
                replay_esn->seq = seq;
        } else {
                diff = replay_esn->seq - seq;

                if (pos >= diff)
                        bitnr = (pos - diff) % replay_esn->replay_window;
                else
                        bitnr = replay_esn->replay_window - (diff - pos);
        }

        nr = bitnr >> 5;
        bitnr = bitnr & 0x1F;
        replay_esn->bmp[nr] |= (1U << bitnr);

        if (xfrm_aevent_is_on(xs_net(x)))
                xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
}

static void xfrm_replay_notify_bmp(struct xfrm_state *x, int event)
{
        struct km_event c;
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
        struct xfrm_replay_state_esn *preplay_esn = x->preplay_esn;

        /* we send notify messages in case
         *  1. we updated on of the sequence numbers, and the seqno difference
         *     is at least x->replay_maxdiff, in this case we also update the
         *     timeout of our timer function
         *  2. if x->replay_maxage has elapsed since last update,
         *     and there were changes
         *
         *  The state structure must be locked!
         */

        switch (event) {
        case XFRM_REPLAY_UPDATE:
                if (!x->replay_maxdiff ||
                    ((replay_esn->seq - preplay_esn->seq < x->replay_maxdiff) &&
                    (replay_esn->oseq - preplay_esn->oseq
                     < x->replay_maxdiff))) {
                        if (x->xflags & XFRM_TIME_DEFER)
                                event = XFRM_REPLAY_TIMEOUT;
                        else
                                return;
                }

                break;

        case XFRM_REPLAY_TIMEOUT:
                if (memcmp(x->replay_esn, x->preplay_esn,
                           xfrm_replay_state_esn_len(replay_esn)) == 0) {
                        x->xflags |= XFRM_TIME_DEFER;
                        return;
                }

                break;
        }

        memcpy(x->preplay_esn, x->replay_esn,
               xfrm_replay_state_esn_len(replay_esn));
        c.event = XFRM_MSG_NEWAE;
        c.data.aevent = event;
        km_state_notify(x, &c);

        if (x->replay_maxage &&
            !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
                x->xflags &= ~XFRM_TIME_DEFER;
}

static void xfrm_replay_notify_esn(struct xfrm_state *x, int event)
{
        u32 seq_diff, oseq_diff;
        struct km_event c;
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
        struct xfrm_replay_state_esn *preplay_esn = x->preplay_esn;

        /* we send notify messages in case
         *  1. we updated on of the sequence numbers, and the seqno difference
         *     is at least x->replay_maxdiff, in this case we also update the
         *     timeout of our timer function
         *  2. if x->replay_maxage has elapsed since last update,
         *     and there were changes
         *
         *  The state structure must be locked!
         */

        switch (event) {
        case XFRM_REPLAY_UPDATE:
                if (x->replay_maxdiff) {
                        if (replay_esn->seq_hi == preplay_esn->seq_hi)
                                seq_diff = replay_esn->seq - preplay_esn->seq;
                        else
                                seq_diff = ~preplay_esn->seq + replay_esn->seq
                                           + 1;

                        if (replay_esn->oseq_hi == preplay_esn->oseq_hi)
                                oseq_diff = replay_esn->oseq
                                            - preplay_esn->oseq;
                        else
                                oseq_diff = ~preplay_esn->oseq
                                            + replay_esn->oseq + 1;

                        if (seq_diff >= x->replay_maxdiff ||
                            oseq_diff >= x->replay_maxdiff)
                                break;
                }

                if (x->xflags & XFRM_TIME_DEFER)
                        event = XFRM_REPLAY_TIMEOUT;
                else
                        return;

                break;

        case XFRM_REPLAY_TIMEOUT:
                if (memcmp(x->replay_esn, x->preplay_esn,
                           xfrm_replay_state_esn_len(replay_esn)) == 0) {
                        x->xflags |= XFRM_TIME_DEFER;
                        return;
                }

                break;
        }

        memcpy(x->preplay_esn, x->replay_esn,
               xfrm_replay_state_esn_len(replay_esn));
        c.event = XFRM_MSG_NEWAE;
        c.data.aevent = event;
        km_state_notify(x, &c);

        if (x->replay_maxage &&
            !mod_timer(&x->rtimer, jiffies + x->replay_maxage))
                x->xflags &= ~XFRM_TIME_DEFER;
}

static int xfrm_replay_overflow_esn(struct xfrm_state *x, struct sk_buff *skb)
{
        int err = 0;
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
        struct net *net = xs_net(x);

        if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
                XFRM_SKB_CB(skb)->seq.output.low = ++replay_esn->oseq;
                XFRM_SKB_CB(skb)->seq.output.hi = replay_esn->oseq_hi;

                if (unlikely(replay_esn->oseq == 0)) {
                        XFRM_SKB_CB(skb)->seq.output.hi = ++replay_esn->oseq_hi;

                        if (replay_esn->oseq_hi == 0) {
                                replay_esn->oseq--;
                                replay_esn->oseq_hi--;
                                xfrm_audit_state_replay_overflow(x, skb);
                                err = -EOVERFLOW;

                                return err;
                        }
                }
                if (xfrm_aevent_is_on(net))
                        xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
        }

        return err;
}

static int xfrm_replay_check_esn(struct xfrm_state *x,
                                 struct sk_buff *skb, __be32 net_seq)
{
        unsigned int bitnr, nr;
        u32 diff;
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
        u32 pos;
        u32 seq = ntohl(net_seq);
        u32 wsize = replay_esn->replay_window;
        u32 top = replay_esn->seq;
        u32 bottom = top - wsize + 1;

        if (!wsize)
                return 0;

        if (unlikely(seq == 0 && replay_esn->seq_hi == 0 &&
                     (replay_esn->seq < replay_esn->replay_window - 1)))
                goto err;

        diff = top - seq;

        if (likely(top >= wsize - 1)) {
                /* A. same subspace */
                if (likely(seq > top) || seq < bottom)
                        return 0;
        } else {
                /* B. window spans two subspaces */
                if (likely(seq > top && seq < bottom))
                        return 0;
                if (seq >= bottom)
                        diff = ~seq + top + 1;
        }

        if (diff >= replay_esn->replay_window) {
                x->stats.replay_window++;
                goto err;
        }

        pos = (replay_esn->seq - 1) % replay_esn->replay_window;

        if (pos >= diff)
                bitnr = (pos - diff) % replay_esn->replay_window;
        else
                bitnr = replay_esn->replay_window - (diff - pos);

        nr = bitnr >> 5;
        bitnr = bitnr & 0x1F;
        if (replay_esn->bmp[nr] & (1U << bitnr))
                goto err_replay;

        return 0;

err_replay:
        x->stats.replay++;
err:
        xfrm_audit_state_replay(x, skb, net_seq);
        return -EINVAL;
}

int xfrm_replay_check(struct xfrm_state *x,
                      struct sk_buff *skb, __be32 net_seq)
{
        switch (x->repl_mode) {
        case XFRM_REPLAY_MODE_LEGACY:
                break;
        case XFRM_REPLAY_MODE_BMP:
                return xfrm_replay_check_bmp(x, skb, net_seq);
        case XFRM_REPLAY_MODE_ESN:
                return xfrm_replay_check_esn(x, skb, net_seq);
        }

        return xfrm_replay_check_legacy(x, skb, net_seq);
}

static int xfrm_replay_recheck_esn(struct xfrm_state *x,
                                   struct sk_buff *skb, __be32 net_seq)
{
        if (unlikely(XFRM_SKB_CB(skb)->seq.input.hi !=
                     htonl(xfrm_replay_seqhi(x, net_seq)))) {
                        x->stats.replay_window++;
                        return -EINVAL;
        }

        return xfrm_replay_check_esn(x, skb, net_seq);
}

int xfrm_replay_recheck(struct xfrm_state *x,
                        struct sk_buff *skb, __be32 net_seq)
{
        switch (x->repl_mode) {
        case XFRM_REPLAY_MODE_LEGACY:
                break;
        case XFRM_REPLAY_MODE_BMP:
                /* no special recheck treatment */
                return xfrm_replay_check_bmp(x, skb, net_seq);
        case XFRM_REPLAY_MODE_ESN:
                return xfrm_replay_recheck_esn(x, skb, net_seq);
        }

        return xfrm_replay_check_legacy(x, skb, net_seq);
}

static void xfrm_replay_advance_esn(struct xfrm_state *x, __be32 net_seq)
{
        unsigned int bitnr, nr, i;
        int wrap;
        u32 diff, pos, seq, seq_hi;
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;

        if (!replay_esn->replay_window)
                return;

        seq = ntohl(net_seq);
        pos = (replay_esn->seq - 1) % replay_esn->replay_window;
        seq_hi = xfrm_replay_seqhi(x, net_seq);
        wrap = seq_hi - replay_esn->seq_hi;

        if ((!wrap && seq > replay_esn->seq) || wrap > 0) {
                if (likely(!wrap))
                        diff = seq - replay_esn->seq;
                else
                        diff = ~replay_esn->seq + seq + 1;

                if (diff < replay_esn->replay_window) {
                        for (i = 1; i < diff; i++) {
                                bitnr = (pos + i) % replay_esn->replay_window;
                                nr = bitnr >> 5;
                                bitnr = bitnr & 0x1F;
                                replay_esn->bmp[nr] &=  ~(1U << bitnr);
                        }
                } else {
                        nr = (replay_esn->replay_window - 1) >> 5;
                        for (i = 0; i <= nr; i++)
                                replay_esn->bmp[i] = 0;
                }

                bitnr = (pos + diff) % replay_esn->replay_window;
                replay_esn->seq = seq;

                if (unlikely(wrap > 0))
                        replay_esn->seq_hi++;
        } else {
                diff = replay_esn->seq - seq;

                if (pos >= diff)
                        bitnr = (pos - diff) % replay_esn->replay_window;
                else
                        bitnr = replay_esn->replay_window - (diff - pos);
        }

        xfrm_dev_state_advance_esn(x);

        nr = bitnr >> 5;
        bitnr = bitnr & 0x1F;
        replay_esn->bmp[nr] |= (1U << bitnr);

        if (xfrm_aevent_is_on(xs_net(x)))
                xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
}

#ifdef CONFIG_XFRM_OFFLOAD
static int xfrm_replay_overflow_offload(struct xfrm_state *x, struct sk_buff *skb)
{
        int err = 0;
        struct net *net = xs_net(x);
        struct xfrm_offload *xo = xfrm_offload(skb);
        __u32 oseq = x->replay.oseq;

        if (!xo)
                return __xfrm_replay_overflow(x, skb);

        if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
                if (!skb_is_gso(skb)) {
                        XFRM_SKB_CB(skb)->seq.output.low = ++oseq;
                        xo->seq.low = oseq;
                } else {
                        XFRM_SKB_CB(skb)->seq.output.low = oseq + 1;
                        xo->seq.low = oseq + 1;
                        oseq += skb_shinfo(skb)->gso_segs;
                }

                XFRM_SKB_CB(skb)->seq.output.hi = 0;
                xo->seq.hi = 0;
                if (unlikely(oseq < x->replay.oseq) &&
                    !(x->props.extra_flags & XFRM_SA_XFLAG_OSEQ_MAY_WRAP)) {
                        xfrm_audit_state_replay_overflow(x, skb);
                        err = -EOVERFLOW;

                        return err;
                }

                x->replay.oseq = oseq;

                if (xfrm_aevent_is_on(net))
                        xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
        }

        return err;
}

static int xfrm_replay_overflow_offload_bmp(struct xfrm_state *x, struct sk_buff *skb)
{
        int err = 0;
        struct xfrm_offload *xo = xfrm_offload(skb);
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
        struct net *net = xs_net(x);
        __u32 oseq = replay_esn->oseq;

        if (!xo)
                return xfrm_replay_overflow_bmp(x, skb);

        if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
                if (!skb_is_gso(skb)) {
                        XFRM_SKB_CB(skb)->seq.output.low = ++oseq;
                        xo->seq.low = oseq;
                } else {
                        XFRM_SKB_CB(skb)->seq.output.low = oseq + 1;
                        xo->seq.low = oseq + 1;
                        oseq += skb_shinfo(skb)->gso_segs;
                }

                XFRM_SKB_CB(skb)->seq.output.hi = 0;
                xo->seq.hi = 0;
                if (unlikely(oseq < replay_esn->oseq) &&
                    !(x->props.extra_flags & XFRM_SA_XFLAG_OSEQ_MAY_WRAP)) {
                        xfrm_audit_state_replay_overflow(x, skb);
                        err = -EOVERFLOW;

                        return err;
                } else {
                        replay_esn->oseq = oseq;
                }

                if (xfrm_aevent_is_on(net))
                        xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
        }

        return err;
}

static int xfrm_replay_overflow_offload_esn(struct xfrm_state *x, struct sk_buff *skb)
{
        int err = 0;
        struct xfrm_offload *xo = xfrm_offload(skb);
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
        struct net *net = xs_net(x);
        __u32 oseq = replay_esn->oseq;
        __u32 oseq_hi = replay_esn->oseq_hi;

        if (!xo)
                return xfrm_replay_overflow_esn(x, skb);

        if (x->type->flags & XFRM_TYPE_REPLAY_PROT) {
                if (!skb_is_gso(skb)) {
                        XFRM_SKB_CB(skb)->seq.output.low = ++oseq;
                        XFRM_SKB_CB(skb)->seq.output.hi = oseq_hi;
                        xo->seq.low = oseq;
                        xo->seq.hi = oseq_hi;
                } else {
                        XFRM_SKB_CB(skb)->seq.output.low = oseq + 1;
                        XFRM_SKB_CB(skb)->seq.output.hi = oseq_hi;
                        xo->seq.low = oseq + 1;
                        xo->seq.hi = oseq_hi;
                        oseq += skb_shinfo(skb)->gso_segs;
                }

                if (unlikely(oseq < replay_esn->oseq)) {
                        replay_esn->oseq_hi = ++oseq_hi;
                        if (xo->seq.low < replay_esn->oseq) {
                                XFRM_SKB_CB(skb)->seq.output.hi = oseq_hi;
                                xo->seq.hi = oseq_hi;
                        }
                        if (replay_esn->oseq_hi == 0) {
                                replay_esn->oseq--;
                                replay_esn->oseq_hi--;
                                xfrm_audit_state_replay_overflow(x, skb);
                                err = -EOVERFLOW;

                                return err;
                        }
                }

                replay_esn->oseq = oseq;
                xfrm_dev_state_advance_esn(x);

                if (xfrm_aevent_is_on(net))
                        xfrm_replay_notify(x, XFRM_REPLAY_UPDATE);
        }

        return err;
}

int xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb)
{
        switch (x->repl_mode) {
        case XFRM_REPLAY_MODE_LEGACY:
                break;
        case XFRM_REPLAY_MODE_BMP:
                return xfrm_replay_overflow_offload_bmp(x, skb);
        case XFRM_REPLAY_MODE_ESN:
                return xfrm_replay_overflow_offload_esn(x, skb);
        }

        return xfrm_replay_overflow_offload(x, skb);
}
#else
int xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb)
{
        switch (x->repl_mode) {
        case XFRM_REPLAY_MODE_LEGACY:
                break;
        case XFRM_REPLAY_MODE_BMP:
                return xfrm_replay_overflow_bmp(x, skb);
        case XFRM_REPLAY_MODE_ESN:
                return xfrm_replay_overflow_esn(x, skb);
        }

        return __xfrm_replay_overflow(x, skb);
}
#endif

int xfrm_init_replay(struct xfrm_state *x, struct netlink_ext_ack *extack)
{
        struct xfrm_replay_state_esn *replay_esn = x->replay_esn;

        if (replay_esn) {
                if (replay_esn->replay_window >
                    replay_esn->bmp_len * sizeof(__u32) * 8) {
                        NL_SET_ERR_MSG(extack, "ESN replay window is too large for the chosen bitmap size");
                        return -EINVAL;
                }

                if (x->props.flags & XFRM_STATE_ESN) {
                        if (replay_esn->replay_window == 0 &&
                            (!x->dir || x->dir == XFRM_SA_DIR_IN)) {
                                NL_SET_ERR_MSG(extack, "ESN replay window must be > 0");
                                return -EINVAL;
                        }
                        x->repl_mode = XFRM_REPLAY_MODE_ESN;
                } else {
                        x->repl_mode = XFRM_REPLAY_MODE_BMP;
                }
        } else {
                x->repl_mode = XFRM_REPLAY_MODE_LEGACY;
        }

        return 0;
}
EXPORT_SYMBOL(xfrm_init_replay);



































































































































































































































































































































































  131 



  131 










  131 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2013 HUAWEI
 * Author: Cai Zhiyong <caizhiyong@huawei.com>
 *
 * Read block device partition table from the command line.
 * Typically used for fixed block (eMMC) embedded devices.
 * It has no MBR, so saves storage space. Bootloader can be easily accessed
 * by absolute address of data on the block device.
 * Users can easily change the partition.
 *
 * The format for the command line is just like mtdparts.
 *
 * For further information, see "Documentation/block/cmdline-partition.rst"
 *
 */
#include <linux/blkdev.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include "check.h"


/* partition flags */
#define PF_RDONLY                   0x01 /* Device is read only */
#define PF_POWERUP_LOCK             0x02 /* Always locked after reset */

struct cmdline_subpart {
        char name[BDEVNAME_SIZE]; /* partition name, such as 'rootfs' */
        sector_t from;
        sector_t size;
        int flags;
        struct cmdline_subpart *next_subpart;
};

struct cmdline_parts {
        char name[BDEVNAME_SIZE]; /* block device, such as 'mmcblk0' */
        unsigned int nr_subparts;
        struct cmdline_subpart *subpart;
        struct cmdline_parts *next_parts;
};

static int parse_subpart(struct cmdline_subpart **subpart, char *partdef)
{
        int ret = 0;
        struct cmdline_subpart *new_subpart;

        *subpart = NULL;

        new_subpart = kzalloc(sizeof(struct cmdline_subpart), GFP_KERNEL);
        if (!new_subpart)
                return -ENOMEM;

        if (*partdef == '-') {
                new_subpart->size = (sector_t)(~0ULL);
                partdef++;
        } else {
                new_subpart->size = (sector_t)memparse(partdef, &partdef);
                if (new_subpart->size < (sector_t)PAGE_SIZE) {
                        pr_warn("cmdline partition size is invalid.");
                        ret = -EINVAL;
                        goto fail;
                }
        }

        if (*partdef == '@') {
                partdef++;
                new_subpart->from = (sector_t)memparse(partdef, &partdef);
        } else {
                new_subpart->from = (sector_t)(~0ULL);
        }

        if (*partdef == '(') {
                partdef++;
                char *next = strsep(&partdef, ")");

                if (!next) {
                        pr_warn("cmdline partition format is invalid.");
                        ret = -EINVAL;
                        goto fail;
                }

                strscpy(new_subpart->name, next, sizeof(new_subpart->name));
        } else
                new_subpart->name[0] = '\0';

        new_subpart->flags = 0;

        if (!strncmp(partdef, "ro", 2)) {
                new_subpart->flags |= PF_RDONLY;
                partdef += 2;
        }

        if (!strncmp(partdef, "lk", 2)) {
                new_subpart->flags |= PF_POWERUP_LOCK;
                partdef += 2;
        }

        *subpart = new_subpart;
        return 0;
fail:
        kfree(new_subpart);
        return ret;
}

static void free_subpart(struct cmdline_parts *parts)
{
        struct cmdline_subpart *subpart;

        while (parts->subpart) {
                subpart = parts->subpart;
                parts->subpart = subpart->next_subpart;
                kfree(subpart);
        }
}

static int parse_parts(struct cmdline_parts **parts, char *bdevdef)
{
        int ret = -EINVAL;
        char *next;
        struct cmdline_subpart **next_subpart;
        struct cmdline_parts *newparts;

        *parts = NULL;

        newparts = kzalloc(sizeof(struct cmdline_parts), GFP_KERNEL);
        if (!newparts)
                return -ENOMEM;

        next = strsep(&bdevdef, ":");
        if (!next) {
                pr_warn("cmdline partition has no block device.");
                goto fail;
        }

        strscpy(newparts->name, next, sizeof(newparts->name));
        newparts->nr_subparts = 0;

        next_subpart = &newparts->subpart;

        while ((next = strsep(&bdevdef, ","))) {
                ret = parse_subpart(next_subpart, next);
                if (ret)
                        goto fail;

                newparts->nr_subparts++;
                next_subpart = &(*next_subpart)->next_subpart;
        }

        if (!newparts->subpart) {
                pr_warn("cmdline partition has no valid partition.");
                ret = -EINVAL;
                goto fail;
        }

        *parts = newparts;

        return 0;
fail:
        free_subpart(newparts);
        kfree(newparts);
        return ret;
}

static void cmdline_parts_free(struct cmdline_parts **parts)
{
        struct cmdline_parts *next_parts;

        while (*parts) {
                next_parts = (*parts)->next_parts;
                free_subpart(*parts);
                kfree(*parts);
                *parts = next_parts;
        }
}

static int cmdline_parts_parse(struct cmdline_parts **parts,
                const char *cmdline)
{
        int ret;
        char *buf;
        char *pbuf;
        char *next;
        struct cmdline_parts **next_parts;

        *parts = NULL;

        pbuf = buf = kstrdup(cmdline, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;

        next_parts = parts;

        while ((next = strsep(&pbuf, ";"))) {
                ret = parse_parts(next_parts, next);
                if (ret)
                        goto fail;

                next_parts = &(*next_parts)->next_parts;
        }

        if (!*parts) {
                pr_warn("cmdline partition has no valid partition.");
                ret = -EINVAL;
                goto fail;
        }

        ret = 0;
done:
        kfree(buf);
        return ret;

fail:
        cmdline_parts_free(parts);
        goto done;
}

static struct cmdline_parts *cmdline_parts_find(struct cmdline_parts *parts,
                                         const char *bdev)
{
        while (parts && strncmp(bdev, parts->name, sizeof(parts->name)))
                parts = parts->next_parts;
        return parts;
}

static char *cmdline;
static struct cmdline_parts *bdev_parts;

static int add_part(int slot, struct cmdline_subpart *subpart,
                struct parsed_partitions *state)
{
        struct partition_meta_info *info;
        char tmp[sizeof(info->volname) + 4];

        if (slot >= state->limit)
                return 1;

        put_partition(state, slot, subpart->from >> 9,
                      subpart->size >> 9);

        if (subpart->flags & PF_RDONLY)
                state->parts[slot].flags |= ADDPART_FLAG_READONLY;

        info = &state->parts[slot].info;

        strscpy(info->volname, subpart->name, sizeof(info->volname));

        snprintf(tmp, sizeof(tmp), "(%s)", info->volname);
        strlcat(state->pp_buf, tmp, PAGE_SIZE);

        state->parts[slot].has_info = true;

        return 0;
}

static int cmdline_parts_set(struct cmdline_parts *parts, sector_t disk_size,
                struct parsed_partitions *state)
{
        sector_t from = 0;
        struct cmdline_subpart *subpart;
        int slot = 1;

        for (subpart = parts->subpart; subpart;
             subpart = subpart->next_subpart, slot++) {
                if (subpart->from == (sector_t)(~0ULL))
                        subpart->from = from;
                else
                        from = subpart->from;

                if (from >= disk_size)
                        break;

                if (subpart->size > (disk_size - from))
                        subpart->size = disk_size - from;

                from += subpart->size;

                if (add_part(slot, subpart, state))
                        break;
        }

        return slot;
}

static int __init cmdline_parts_setup(char *s)
{
        cmdline = s;
        return 1;
}
__setup("blkdevparts=", cmdline_parts_setup);

static bool has_overlaps(sector_t from, sector_t size,
                         sector_t from2, sector_t size2)
{
        sector_t end = from + size;
        sector_t end2 = from2 + size2;

        if (from >= from2 && from < end2)
                return true;

        if (end > from2 && end <= end2)
                return true;

        if (from2 >= from && from2 < end)
                return true;

        if (end2 > from && end2 <= end)
                return true;

        return false;
}

static inline void overlaps_warns_header(void)
{
        pr_warn("Overlapping partitions are used in command line partitions.");
        pr_warn("Don't use filesystems on overlapping partitions:");
}

static void cmdline_parts_verifier(int slot, struct parsed_partitions *state)
{
        int i;
        bool header = true;

        for (; slot < state->limit && state->parts[slot].has_info; slot++) {
                for (i = slot+1; i < state->limit && state->parts[i].has_info;
                     i++) {
                        if (has_overlaps(state->parts[slot].from,
                                         state->parts[slot].size,
                                         state->parts[i].from,
                                         state->parts[i].size)) {
                                if (header) {
                                        header = false;
                                        overlaps_warns_header();
                                }
                                pr_warn("%s[%llu,%llu] overlaps with "
                                        "%s[%llu,%llu].",
                                        state->parts[slot].info.volname,
                                        (u64)state->parts[slot].from << 9,
                                        (u64)state->parts[slot].size << 9,
                                        state->parts[i].info.volname,
                                        (u64)state->parts[i].from << 9,
                                        (u64)state->parts[i].size << 9);
                        }
                }
        }
}

/*
 * Purpose: allocate cmdline partitions.
 * Returns:
 * -1 if unable to read the partition table
 *  0 if this isn't our partition table
 *  1 if successful
 */
int cmdline_partition(struct parsed_partitions *state)
{
        sector_t disk_size;
        struct cmdline_parts *parts;

        if (cmdline) {
                if (bdev_parts)
                        cmdline_parts_free(&bdev_parts);

                if (cmdline_parts_parse(&bdev_parts, cmdline)) {
                        cmdline = NULL;
                        return -1;
                }
                cmdline = NULL;
        }

        if (!bdev_parts)
                return 0;

        parts = cmdline_parts_find(bdev_parts, state->disk->disk_name);
        if (!parts)
                return 0;

        disk_size = get_capacity(state->disk) << 9;

        cmdline_parts_set(parts, disk_size, state);
        cmdline_parts_verifier(1, state);

        strlcat(state->pp_buf, "\n", PAGE_SIZE);

        return 1;
}
































































































































































    2 
    5 





    2 
    5 



























    7 

















































































































































































































    7 




    7 


    1 









    5 

    1 



    1 

    6 



    7 

    7 









    7 



    7 



















    7 








   20 
    1 

   20 





    7 






    7 








    6 
    1 

    6 


















    2 




   21 
    1 





    1 











   18 











    3 






    2 
    1 













   21 





   10 

   11 















    7 







    7 




    7 


















    1 




    7 


    3 

    2 
    2 
















    4 
    3 
    4 











    3 
    4 
    6 




    4 
    3 




    7 




















    7 


    7 





   19 



   18 



    3 


    3 

    1 
    1 
   13 





   14 


   13 








   19 
   14 


   18 
   13 









   47 




   47 








































   47 

































   18 
   37 







   47 






   23 


   31 
    1 




   31 






    6 

   24 






   30 














   30 



    7 




   23 












    2 





   11 
    8 
   11 





    1 


    1 


    6 



    7 


    7 




















    7 






    6 
   17 

   40 







    7 













    7 




    7 










    7 







   43 


















   33 



   21 







   48 

   48 







































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
/*
 * cdc_ncm.c
 *
 * Copyright (C) ST-Ericsson 2010-2012
 * Contact: Alexey Orishko <alexey.orishko@stericsson.com>
 * Original author: Hans Petter Selasky <hans.petter.selasky@stericsson.com>
 *
 * USB Host Driver for Network Control Model (NCM)
 * http://www.usb.org/developers/docs/devclass_docs/NCM10_012011.zip
 *
 * The NCM encoding, decoding and initialization logic
 * derives from FreeBSD 8.x. if_cdce.c and if_cdcereg.h
 *
 * This software is available to you under a choice of one of two
 * licenses. You may choose this file to be licensed under the terms
 * of the GNU General Public License (GPL) Version 2 or the 2-clause
 * BSD license listed below:
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/ctype.h>
#include <linux/etherdevice.h>
#include <linux/ethtool.h>
#include <linux/kstrtox.h>
#include <linux/workqueue.h>
#include <linux/mii.h>
#include <linux/crc32.h>
#include <linux/usb.h>
#include <linux/hrtimer.h>
#include <linux/atomic.h>
#include <linux/usb/usbnet.h>
#include <linux/usb/cdc.h>
#include <linux/usb/cdc_ncm.h>

#if IS_ENABLED(CONFIG_USB_NET_CDC_MBIM)
static bool prefer_mbim = true;
#else
static bool prefer_mbim;
#endif
module_param(prefer_mbim, bool, 0644);
MODULE_PARM_DESC(prefer_mbim, "Prefer MBIM setting on dual NCM/MBIM functions");

static void cdc_ncm_txpath_bh(struct tasklet_struct *t);
static void cdc_ncm_tx_timeout_start(struct cdc_ncm_ctx *ctx);
static enum hrtimer_restart cdc_ncm_tx_timer_cb(struct hrtimer *hr_timer);
static struct usb_driver cdc_ncm_driver;

struct cdc_ncm_stats {
        char stat_string[ETH_GSTRING_LEN];
        int sizeof_stat;
        int stat_offset;
};

#define CDC_NCM_STAT(str, m) { \
                .stat_string = str, \
                .sizeof_stat = sizeof(((struct cdc_ncm_ctx *)0)->m), \
                .stat_offset = offsetof(struct cdc_ncm_ctx, m) }
#define CDC_NCM_SIMPLE_STAT(m)        CDC_NCM_STAT(__stringify(m), m)

static const struct cdc_ncm_stats cdc_ncm_gstrings_stats[] = {
        CDC_NCM_SIMPLE_STAT(tx_reason_ntb_full),
        CDC_NCM_SIMPLE_STAT(tx_reason_ndp_full),
        CDC_NCM_SIMPLE_STAT(tx_reason_timeout),
        CDC_NCM_SIMPLE_STAT(tx_reason_max_datagram),
        CDC_NCM_SIMPLE_STAT(tx_overhead),
        CDC_NCM_SIMPLE_STAT(tx_ntbs),
        CDC_NCM_SIMPLE_STAT(rx_overhead),
        CDC_NCM_SIMPLE_STAT(rx_ntbs),
};

#define CDC_NCM_LOW_MEM_MAX_CNT 10

static int cdc_ncm_get_sset_count(struct net_device __always_unused *netdev, int sset)
{
        switch (sset) {
        case ETH_SS_STATS:
                return ARRAY_SIZE(cdc_ncm_gstrings_stats);
        default:
                return -EOPNOTSUPP;
        }
}

static void cdc_ncm_get_ethtool_stats(struct net_device *netdev,
                                    struct ethtool_stats __always_unused *stats,
                                    u64 *data)
{
        struct usbnet *dev = netdev_priv(netdev);
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        int i;
        char *p = NULL;

        for (i = 0; i < ARRAY_SIZE(cdc_ncm_gstrings_stats); i++) {
                p = (char *)ctx + cdc_ncm_gstrings_stats[i].stat_offset;
                data[i] = (cdc_ncm_gstrings_stats[i].sizeof_stat == sizeof(u64)) ? *(u64 *)p : *(u32 *)p;
        }
}

static void cdc_ncm_get_strings(struct net_device __always_unused *netdev, u32 stringset, u8 *data)
{
        u8 *p = data;
        int i;

        switch (stringset) {
        case ETH_SS_STATS:
                for (i = 0; i < ARRAY_SIZE(cdc_ncm_gstrings_stats); i++) {
                        memcpy(p, cdc_ncm_gstrings_stats[i].stat_string, ETH_GSTRING_LEN);
                        p += ETH_GSTRING_LEN;
                }
        }
}

static void cdc_ncm_update_rxtx_max(struct usbnet *dev, u32 new_rx, u32 new_tx);

static const struct ethtool_ops cdc_ncm_ethtool_ops = {
        .get_link                = usbnet_get_link,
        .nway_reset                = usbnet_nway_reset,
        .get_drvinfo                = usbnet_get_drvinfo,
        .get_msglevel                = usbnet_get_msglevel,
        .set_msglevel                = usbnet_set_msglevel,
        .get_ts_info                = ethtool_op_get_ts_info,
        .get_sset_count                = cdc_ncm_get_sset_count,
        .get_strings                = cdc_ncm_get_strings,
        .get_ethtool_stats        = cdc_ncm_get_ethtool_stats,
        .get_link_ksettings        = usbnet_get_link_ksettings_internal,
        .set_link_ksettings        = NULL,
};

static u32 cdc_ncm_check_rx_max(struct usbnet *dev, u32 new_rx)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        u32 val, max, min;

        /* clamp new_rx to sane values */
        min = USB_CDC_NCM_NTB_MIN_IN_SIZE;
        max = min_t(u32, CDC_NCM_NTB_MAX_SIZE_RX, le32_to_cpu(ctx->ncm_parm.dwNtbInMaxSize));

        /* dwNtbInMaxSize spec violation? Use MIN size for both limits */
        if (max < min) {
                dev_warn(&dev->intf->dev, "dwNtbInMaxSize=%u is too small. Using %u\n",
                         le32_to_cpu(ctx->ncm_parm.dwNtbInMaxSize), min);
                max = min;
        }

        val = clamp_t(u32, new_rx, min, max);
        if (val != new_rx)
                dev_dbg(&dev->intf->dev, "rx_max must be in the [%u, %u] range\n", min, max);

        return val;
}

static u32 cdc_ncm_check_tx_max(struct usbnet *dev, u32 new_tx)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        u32 val, max, min;

        /* clamp new_tx to sane values */
        if (ctx->is_ndp16)
                min = ctx->max_datagram_size + ctx->max_ndp_size + sizeof(struct usb_cdc_ncm_nth16);
        else
                min = ctx->max_datagram_size + ctx->max_ndp_size + sizeof(struct usb_cdc_ncm_nth32);

        if (le32_to_cpu(ctx->ncm_parm.dwNtbOutMaxSize) == 0)
                max = CDC_NCM_NTB_MAX_SIZE_TX; /* dwNtbOutMaxSize not set */
        else
                max = clamp_t(u32, le32_to_cpu(ctx->ncm_parm.dwNtbOutMaxSize),
                              USB_CDC_NCM_NTB_MIN_OUT_SIZE,
                              CDC_NCM_NTB_MAX_SIZE_TX);

        /* some devices set dwNtbOutMaxSize too low for the above default */
        min = min(min, max);

        val = clamp_t(u32, new_tx, min, max);
        if (val != new_tx)
                dev_dbg(&dev->intf->dev, "tx_max must be in the [%u, %u] range\n", min, max);

        return val;
}

static ssize_t min_tx_pkt_show(struct device *d,
                               struct device_attribute *attr, char *buf)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];

        return sprintf(buf, "%u\n", ctx->min_tx_pkt);
}

static ssize_t rx_max_show(struct device *d,
                           struct device_attribute *attr, char *buf)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];

        return sprintf(buf, "%u\n", ctx->rx_max);
}

static ssize_t tx_max_show(struct device *d,
                           struct device_attribute *attr, char *buf)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];

        return sprintf(buf, "%u\n", ctx->tx_max);
}

static ssize_t tx_timer_usecs_show(struct device *d,
                                   struct device_attribute *attr, char *buf)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];

        return sprintf(buf, "%u\n", ctx->timer_interval / (u32)NSEC_PER_USEC);
}

static ssize_t min_tx_pkt_store(struct device *d,
                                struct device_attribute *attr,
                                const char *buf, size_t len)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        unsigned long val;

        /* no need to restrict values - anything from 0 to infinity is OK */
        if (kstrtoul(buf, 0, &val))
                return -EINVAL;

        ctx->min_tx_pkt = val;
        return len;
}

static ssize_t rx_max_store(struct device *d,
                            struct device_attribute *attr,
                            const char *buf, size_t len)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        unsigned long val;

        if (kstrtoul(buf, 0, &val) || cdc_ncm_check_rx_max(dev, val) != val)
                return -EINVAL;

        cdc_ncm_update_rxtx_max(dev, val, ctx->tx_max);
        return len;
}

static ssize_t tx_max_store(struct device *d,
                            struct device_attribute *attr,
                            const char *buf, size_t len)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        unsigned long val;

        if (kstrtoul(buf, 0, &val) || cdc_ncm_check_tx_max(dev, val) != val)
                return -EINVAL;

        cdc_ncm_update_rxtx_max(dev, ctx->rx_max, val);
        return len;
}

static ssize_t tx_timer_usecs_store(struct device *d,
                                    struct device_attribute *attr,
                                    const char *buf, size_t len)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        ssize_t ret;
        unsigned long val;

        ret = kstrtoul(buf, 0, &val);
        if (ret)
                return ret;
        if (val && (val < CDC_NCM_TIMER_INTERVAL_MIN || val > CDC_NCM_TIMER_INTERVAL_MAX))
                return -EINVAL;

        spin_lock_bh(&ctx->mtx);
        ctx->timer_interval = val * NSEC_PER_USEC;
        if (!ctx->timer_interval)
                ctx->tx_timer_pending = 0;
        spin_unlock_bh(&ctx->mtx);
        return len;
}

static DEVICE_ATTR_RW(min_tx_pkt);
static DEVICE_ATTR_RW(rx_max);
static DEVICE_ATTR_RW(tx_max);
static DEVICE_ATTR_RW(tx_timer_usecs);

static ssize_t ndp_to_end_show(struct device *d, struct device_attribute *attr, char *buf)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];

        return sprintf(buf, "%c\n", ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END ? 'Y' : 'N');
}

static ssize_t ndp_to_end_store(struct device *d,  struct device_attribute *attr, const char *buf, size_t len)
{
        struct usbnet *dev = netdev_priv(to_net_dev(d));
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        bool enable;

        if (kstrtobool(buf, &enable))
                return -EINVAL;

        /* no change? */
        if (enable == (ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END))
                return len;

        if (enable) {
                if (ctx->is_ndp16 && !ctx->delayed_ndp16) {
                        ctx->delayed_ndp16 = kzalloc(ctx->max_ndp_size, GFP_KERNEL);
                        if (!ctx->delayed_ndp16)
                                return -ENOMEM;
                }
                if (!ctx->is_ndp16 && !ctx->delayed_ndp32) {
                        ctx->delayed_ndp32 = kzalloc(ctx->max_ndp_size, GFP_KERNEL);
                        if (!ctx->delayed_ndp32)
                                return -ENOMEM;
                }
        }

        /* flush pending data before changing flag */
        netif_tx_lock_bh(dev->net);
        usbnet_start_xmit(NULL, dev->net);
        spin_lock_bh(&ctx->mtx);
        if (enable)
                ctx->drvflags |= CDC_NCM_FLAG_NDP_TO_END;
        else
                ctx->drvflags &= ~CDC_NCM_FLAG_NDP_TO_END;
        spin_unlock_bh(&ctx->mtx);
        netif_tx_unlock_bh(dev->net);

        return len;
}
static DEVICE_ATTR_RW(ndp_to_end);

#define NCM_PARM_ATTR(name, format, tocpu)                                \
static ssize_t cdc_ncm_show_##name(struct device *d, struct device_attribute *attr, char *buf) \
{ \
        struct usbnet *dev = netdev_priv(to_net_dev(d)); \
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0]; \
        return sprintf(buf, format "\n", tocpu(ctx->ncm_parm.name));        \
} \
static DEVICE_ATTR(name, 0444, cdc_ncm_show_##name, NULL)

NCM_PARM_ATTR(bmNtbFormatsSupported, "0x%04x", le16_to_cpu);
NCM_PARM_ATTR(dwNtbInMaxSize, "%u", le32_to_cpu);
NCM_PARM_ATTR(wNdpInDivisor, "%u", le16_to_cpu);
NCM_PARM_ATTR(wNdpInPayloadRemainder, "%u", le16_to_cpu);
NCM_PARM_ATTR(wNdpInAlignment, "%u", le16_to_cpu);
NCM_PARM_ATTR(dwNtbOutMaxSize, "%u", le32_to_cpu);
NCM_PARM_ATTR(wNdpOutDivisor, "%u", le16_to_cpu);
NCM_PARM_ATTR(wNdpOutPayloadRemainder, "%u", le16_to_cpu);
NCM_PARM_ATTR(wNdpOutAlignment, "%u", le16_to_cpu);
NCM_PARM_ATTR(wNtbOutMaxDatagrams, "%u", le16_to_cpu);

static struct attribute *cdc_ncm_sysfs_attrs[] = {
        &dev_attr_min_tx_pkt.attr,
        &dev_attr_ndp_to_end.attr,
        &dev_attr_rx_max.attr,
        &dev_attr_tx_max.attr,
        &dev_attr_tx_timer_usecs.attr,
        &dev_attr_bmNtbFormatsSupported.attr,
        &dev_attr_dwNtbInMaxSize.attr,
        &dev_attr_wNdpInDivisor.attr,
        &dev_attr_wNdpInPayloadRemainder.attr,
        &dev_attr_wNdpInAlignment.attr,
        &dev_attr_dwNtbOutMaxSize.attr,
        &dev_attr_wNdpOutDivisor.attr,
        &dev_attr_wNdpOutPayloadRemainder.attr,
        &dev_attr_wNdpOutAlignment.attr,
        &dev_attr_wNtbOutMaxDatagrams.attr,
        NULL,
};

static const struct attribute_group cdc_ncm_sysfs_attr_group = {
        .name = "cdc_ncm",
        .attrs = cdc_ncm_sysfs_attrs,
};

/* handle rx_max and tx_max changes */
static void cdc_ncm_update_rxtx_max(struct usbnet *dev, u32 new_rx, u32 new_tx)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        u8 iface_no = ctx->control->cur_altsetting->desc.bInterfaceNumber;
        u32 val;

        val = cdc_ncm_check_rx_max(dev, new_rx);

        /* inform device about NTB input size changes */
        if (val != ctx->rx_max) {
                __le32 dwNtbInMaxSize = cpu_to_le32(val);

                dev_info(&dev->intf->dev, "setting rx_max = %u\n", val);

                /* tell device to use new size */
                if (usbnet_write_cmd(dev, USB_CDC_SET_NTB_INPUT_SIZE,
                                     USB_TYPE_CLASS | USB_DIR_OUT
                                     | USB_RECIP_INTERFACE,
                                     0, iface_no, &dwNtbInMaxSize, 4) < 0)
                        dev_dbg(&dev->intf->dev, "Setting NTB Input Size failed\n");
                else
                        ctx->rx_max = val;
        }

        /* usbnet use these values for sizing rx queues */
        if (dev->rx_urb_size != ctx->rx_max) {
                dev->rx_urb_size = ctx->rx_max;
                if (netif_running(dev->net))
                        usbnet_unlink_rx_urbs(dev);
        }

        val = cdc_ncm_check_tx_max(dev, new_tx);
        if (val != ctx->tx_max)
                dev_info(&dev->intf->dev, "setting tx_max = %u\n", val);

        /* Adding a pad byte here if necessary simplifies the handling
         * in cdc_ncm_fill_tx_frame, making tx_max always represent
         * the real skb max size.
         *
         * We cannot use dev->maxpacket here because this is called from
         * .bind which is called before usbnet sets up dev->maxpacket
         */
        if (val != le32_to_cpu(ctx->ncm_parm.dwNtbOutMaxSize) &&
            val % usb_maxpacket(dev->udev, dev->out) == 0)
                val++;

        /* we might need to flush any pending tx buffers if running */
        if (netif_running(dev->net) && val > ctx->tx_max) {
                netif_tx_lock_bh(dev->net);
                usbnet_start_xmit(NULL, dev->net);
                /* make sure tx_curr_skb is reallocated if it was empty */
                if (ctx->tx_curr_skb) {
                        dev_kfree_skb_any(ctx->tx_curr_skb);
                        ctx->tx_curr_skb = NULL;
                }
                ctx->tx_max = val;
                netif_tx_unlock_bh(dev->net);
        } else {
                ctx->tx_max = val;
        }

        dev->hard_mtu = ctx->tx_max;

        /* max qlen depend on hard_mtu and rx_urb_size */
        usbnet_update_max_qlen(dev);

        /* never pad more than 3 full USB packets per transfer */
        ctx->min_tx_pkt = clamp_t(u16, ctx->tx_max - 3 * usb_maxpacket(dev->udev, dev->out),
                                  CDC_NCM_MIN_TX_PKT, ctx->tx_max);
}

/* helpers for NCM and MBIM differences */
static u8 cdc_ncm_flags(struct usbnet *dev)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];

        if (cdc_ncm_comm_intf_is_mbim(dev->intf->cur_altsetting) && ctx->mbim_desc)
                return ctx->mbim_desc->bmNetworkCapabilities;
        if (ctx->func_desc)
                return ctx->func_desc->bmNetworkCapabilities;
        return 0;
}

static int cdc_ncm_eth_hlen(struct usbnet *dev)
{
        if (cdc_ncm_comm_intf_is_mbim(dev->intf->cur_altsetting))
                return 0;
        return ETH_HLEN;
}

static u32 cdc_ncm_min_dgram_size(struct usbnet *dev)
{
        if (cdc_ncm_comm_intf_is_mbim(dev->intf->cur_altsetting))
                return CDC_MBIM_MIN_DATAGRAM_SIZE;
        return CDC_NCM_MIN_DATAGRAM_SIZE;
}

static u32 cdc_ncm_max_dgram_size(struct usbnet *dev)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];

        if (cdc_ncm_comm_intf_is_mbim(dev->intf->cur_altsetting) && ctx->mbim_desc)
                return le16_to_cpu(ctx->mbim_desc->wMaxSegmentSize);
        if (ctx->ether_desc)
                return le16_to_cpu(ctx->ether_desc->wMaxSegmentSize);
        return CDC_NCM_MAX_DATAGRAM_SIZE;
}

/* initial one-time device setup.  MUST be called with the data interface
 * in altsetting 0
 */
static int cdc_ncm_init(struct usbnet *dev)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        u8 iface_no = ctx->control->cur_altsetting->desc.bInterfaceNumber;
        int err;

        err = usbnet_read_cmd(dev, USB_CDC_GET_NTB_PARAMETERS,
                              USB_TYPE_CLASS | USB_DIR_IN
                              |USB_RECIP_INTERFACE,
                              0, iface_no, &ctx->ncm_parm,
                              sizeof(ctx->ncm_parm));
        if (err < 0) {
                dev_err(&dev->intf->dev, "failed GET_NTB_PARAMETERS\n");
                return err; /* GET_NTB_PARAMETERS is required */
        }

        /* set CRC Mode */
        if (cdc_ncm_flags(dev) & USB_CDC_NCM_NCAP_CRC_MODE) {
                dev_dbg(&dev->intf->dev, "Setting CRC mode off\n");
                err = usbnet_write_cmd(dev, USB_CDC_SET_CRC_MODE,
                                       USB_TYPE_CLASS | USB_DIR_OUT
                                       | USB_RECIP_INTERFACE,
                                       USB_CDC_NCM_CRC_NOT_APPENDED,
                                       iface_no, NULL, 0);
                if (err < 0)
                        dev_err(&dev->intf->dev, "SET_CRC_MODE failed\n");
        }

        /* use ndp16 by default */
        ctx->is_ndp16 = 1;

        /* set NTB format, if both formats are supported.
         *
         * "The host shall only send this command while the NCM Data
         *  Interface is in alternate setting 0."
         */
        if (le16_to_cpu(ctx->ncm_parm.bmNtbFormatsSupported) &
                                                USB_CDC_NCM_NTB32_SUPPORTED) {
                if (ctx->drvflags & CDC_NCM_FLAG_PREFER_NTB32) {
                        ctx->is_ndp16 = 0;
                        dev_dbg(&dev->intf->dev, "Setting NTB format to 32-bit\n");
                        err = usbnet_write_cmd(dev, USB_CDC_SET_NTB_FORMAT,
                                               USB_TYPE_CLASS | USB_DIR_OUT
                                               | USB_RECIP_INTERFACE,
                                               USB_CDC_NCM_NTB32_FORMAT,
                                               iface_no, NULL, 0);
                } else {
                        ctx->is_ndp16 = 1;
                        dev_dbg(&dev->intf->dev, "Setting NTB format to 16-bit\n");
                        err = usbnet_write_cmd(dev, USB_CDC_SET_NTB_FORMAT,
                                               USB_TYPE_CLASS | USB_DIR_OUT
                                               | USB_RECIP_INTERFACE,
                                               USB_CDC_NCM_NTB16_FORMAT,
                                               iface_no, NULL, 0);
                }
                if (err < 0) {
                        ctx->is_ndp16 = 1;
                        dev_err(&dev->intf->dev, "SET_NTB_FORMAT failed\n");
                }
        }

        /* set initial device values */
        ctx->rx_max = le32_to_cpu(ctx->ncm_parm.dwNtbInMaxSize);
        ctx->tx_max = le32_to_cpu(ctx->ncm_parm.dwNtbOutMaxSize);
        ctx->tx_remainder = le16_to_cpu(ctx->ncm_parm.wNdpOutPayloadRemainder);
        ctx->tx_modulus = le16_to_cpu(ctx->ncm_parm.wNdpOutDivisor);
        ctx->tx_ndp_modulus = le16_to_cpu(ctx->ncm_parm.wNdpOutAlignment);
        /* devices prior to NCM Errata shall set this field to zero */
        ctx->tx_max_datagrams = le16_to_cpu(ctx->ncm_parm.wNtbOutMaxDatagrams);

        dev_dbg(&dev->intf->dev,
                "dwNtbInMaxSize=%u dwNtbOutMaxSize=%u wNdpOutPayloadRemainder=%u wNdpOutDivisor=%u wNdpOutAlignment=%u wNtbOutMaxDatagrams=%u flags=0x%x\n",
                ctx->rx_max, ctx->tx_max, ctx->tx_remainder, ctx->tx_modulus,
                ctx->tx_ndp_modulus, ctx->tx_max_datagrams, cdc_ncm_flags(dev));

        /* max count of tx datagrams */
        if ((ctx->tx_max_datagrams == 0) ||
                        (ctx->tx_max_datagrams > CDC_NCM_DPT_DATAGRAMS_MAX))
                ctx->tx_max_datagrams = CDC_NCM_DPT_DATAGRAMS_MAX;

        /* set up maximum NDP size */
        if (ctx->is_ndp16)
                ctx->max_ndp_size = sizeof(struct usb_cdc_ncm_ndp16) + (ctx->tx_max_datagrams + 1) * sizeof(struct usb_cdc_ncm_dpe16);
        else
                ctx->max_ndp_size = sizeof(struct usb_cdc_ncm_ndp32) + (ctx->tx_max_datagrams + 1) * sizeof(struct usb_cdc_ncm_dpe32);

        /* initial coalescing timer interval */
        ctx->timer_interval = CDC_NCM_TIMER_INTERVAL_USEC * NSEC_PER_USEC;

        return 0;
}

/* set a new max datagram size */
static void cdc_ncm_set_dgram_size(struct usbnet *dev, int new_size)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        u8 iface_no = ctx->control->cur_altsetting->desc.bInterfaceNumber;
        __le16 max_datagram_size;
        u16 mbim_mtu;
        int err;

        /* set default based on descriptors */
        ctx->max_datagram_size = clamp_t(u32, new_size,
                                         cdc_ncm_min_dgram_size(dev),
                                         CDC_NCM_MAX_DATAGRAM_SIZE);

        /* inform the device about the selected Max Datagram Size? */
        if (!(cdc_ncm_flags(dev) & USB_CDC_NCM_NCAP_MAX_DATAGRAM_SIZE))
                goto out;

        /* read current mtu value from device */
        err = usbnet_read_cmd(dev, USB_CDC_GET_MAX_DATAGRAM_SIZE,
                              USB_TYPE_CLASS | USB_DIR_IN | USB_RECIP_INTERFACE,
                              0, iface_no, &max_datagram_size, sizeof(max_datagram_size));
        if (err != sizeof(max_datagram_size)) {
                dev_dbg(&dev->intf->dev, "GET_MAX_DATAGRAM_SIZE failed\n");
                goto out;
        }

        if (le16_to_cpu(max_datagram_size) == ctx->max_datagram_size)
                goto out;

        max_datagram_size = cpu_to_le16(ctx->max_datagram_size);
        err = usbnet_write_cmd(dev, USB_CDC_SET_MAX_DATAGRAM_SIZE,
                               USB_TYPE_CLASS | USB_DIR_OUT | USB_RECIP_INTERFACE,
                               0, iface_no, &max_datagram_size, sizeof(max_datagram_size));
        if (err < 0)
                dev_dbg(&dev->intf->dev, "SET_MAX_DATAGRAM_SIZE failed\n");

out:
        /* set MTU to max supported by the device if necessary */
        dev->net->mtu = min_t(int, dev->net->mtu, ctx->max_datagram_size - cdc_ncm_eth_hlen(dev));

        /* do not exceed operator preferred MTU */
        if (ctx->mbim_extended_desc) {
                mbim_mtu = le16_to_cpu(ctx->mbim_extended_desc->wMTU);
                if (mbim_mtu != 0 && mbim_mtu < dev->net->mtu)
                        dev->net->mtu = mbim_mtu;
        }
}

static void cdc_ncm_fix_modulus(struct usbnet *dev)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        u32 val;

        /*
         * verify that the structure alignment is:
         * - power of two
         * - not greater than the maximum transmit length
         * - not less than four bytes
         */
        val = ctx->tx_ndp_modulus;

        if ((val < USB_CDC_NCM_NDP_ALIGN_MIN_SIZE) ||
            (val != ((-val) & val)) || (val >= ctx->tx_max)) {
                dev_dbg(&dev->intf->dev, "Using default alignment: 4 bytes\n");
                ctx->tx_ndp_modulus = USB_CDC_NCM_NDP_ALIGN_MIN_SIZE;
        }

        /*
         * verify that the payload alignment is:
         * - power of two
         * - not greater than the maximum transmit length
         * - not less than four bytes
         */
        val = ctx->tx_modulus;

        if ((val < USB_CDC_NCM_NDP_ALIGN_MIN_SIZE) ||
            (val != ((-val) & val)) || (val >= ctx->tx_max)) {
                dev_dbg(&dev->intf->dev, "Using default transmit modulus: 4 bytes\n");
                ctx->tx_modulus = USB_CDC_NCM_NDP_ALIGN_MIN_SIZE;
        }

        /* verify the payload remainder */
        if (ctx->tx_remainder >= ctx->tx_modulus) {
                dev_dbg(&dev->intf->dev, "Using default transmit remainder: 0 bytes\n");
                ctx->tx_remainder = 0;
        }

        /* adjust TX-remainder according to NCM specification. */
        ctx->tx_remainder = ((ctx->tx_remainder - cdc_ncm_eth_hlen(dev)) &
                             (ctx->tx_modulus - 1));
}

static int cdc_ncm_setup(struct usbnet *dev)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        u32 def_rx, def_tx;

        /* be conservative when selecting initial buffer size to
         * increase the number of hosts this will work for
         */
        def_rx = min_t(u32, CDC_NCM_NTB_DEF_SIZE_RX,
                       le32_to_cpu(ctx->ncm_parm.dwNtbInMaxSize));
        def_tx = min_t(u32, CDC_NCM_NTB_DEF_SIZE_TX,
                       le32_to_cpu(ctx->ncm_parm.dwNtbOutMaxSize));

        /* clamp rx_max and tx_max and inform device */
        cdc_ncm_update_rxtx_max(dev, def_rx, def_tx);

        /* sanitize the modulus and remainder values */
        cdc_ncm_fix_modulus(dev);

        /* set max datagram size */
        cdc_ncm_set_dgram_size(dev, cdc_ncm_max_dgram_size(dev));
        return 0;
}

static void
cdc_ncm_find_endpoints(struct usbnet *dev, struct usb_interface *intf)
{
        struct usb_host_endpoint *e, *in = NULL, *out = NULL;
        u8 ep;

        for (ep = 0; ep < intf->cur_altsetting->desc.bNumEndpoints; ep++) {
                e = intf->cur_altsetting->endpoint + ep;

                /* ignore endpoints which cannot transfer data */
                if (!usb_endpoint_maxp(&e->desc))
                        continue;

                switch (e->desc.bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) {
                case USB_ENDPOINT_XFER_INT:
                        if (usb_endpoint_dir_in(&e->desc)) {
                                if (!dev->status)
                                        dev->status = e;
                        }
                        break;

                case USB_ENDPOINT_XFER_BULK:
                        if (usb_endpoint_dir_in(&e->desc)) {
                                if (!in)
                                        in = e;
                        } else {
                                if (!out)
                                        out = e;
                        }
                        break;

                default:
                        break;
                }
        }
        if (in && !dev->in)
                dev->in = usb_rcvbulkpipe(dev->udev,
                                          in->desc.bEndpointAddress &
                                          USB_ENDPOINT_NUMBER_MASK);
        if (out && !dev->out)
                dev->out = usb_sndbulkpipe(dev->udev,
                                           out->desc.bEndpointAddress &
                                           USB_ENDPOINT_NUMBER_MASK);
}

static void cdc_ncm_free(struct cdc_ncm_ctx *ctx)
{
        if (ctx == NULL)
                return;

        if (ctx->tx_rem_skb != NULL) {
                dev_kfree_skb_any(ctx->tx_rem_skb);
                ctx->tx_rem_skb = NULL;
        }

        if (ctx->tx_curr_skb != NULL) {
                dev_kfree_skb_any(ctx->tx_curr_skb);
                ctx->tx_curr_skb = NULL;
        }

        if (ctx->is_ndp16)
                kfree(ctx->delayed_ndp16);
        else
                kfree(ctx->delayed_ndp32);

        kfree(ctx);
}

/* we need to override the usbnet change_mtu ndo for two reasons:
 *  - respect the negotiated maximum datagram size
 *  - avoid unwanted changes to rx and tx buffers
 */
int cdc_ncm_change_mtu(struct net_device *net, int new_mtu)
{
        struct usbnet *dev = netdev_priv(net);

        WRITE_ONCE(net->mtu, new_mtu);
        cdc_ncm_set_dgram_size(dev, new_mtu + cdc_ncm_eth_hlen(dev));

        return 0;
}
EXPORT_SYMBOL_GPL(cdc_ncm_change_mtu);

static const struct net_device_ops cdc_ncm_netdev_ops = {
        .ndo_open             = usbnet_open,
        .ndo_stop             = usbnet_stop,
        .ndo_start_xmit             = usbnet_start_xmit,
        .ndo_tx_timeout             = usbnet_tx_timeout,
        .ndo_set_rx_mode     = usbnet_set_rx_mode,
        .ndo_get_stats64     = dev_get_tstats64,
        .ndo_change_mtu             = cdc_ncm_change_mtu,
        .ndo_set_mac_address = eth_mac_addr,
        .ndo_validate_addr   = eth_validate_addr,
};

int cdc_ncm_bind_common(struct usbnet *dev, struct usb_interface *intf, u8 data_altsetting, int drvflags)
{
        struct cdc_ncm_ctx *ctx;
        struct usb_driver *driver;
        u8 *buf;
        int len;
        int temp;
        u8 iface_no;
        struct usb_cdc_parsed_header hdr;

        ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
        if (!ctx)
                return -ENOMEM;

        ctx->dev = dev;

        hrtimer_setup(&ctx->tx_timer, &cdc_ncm_tx_timer_cb, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
        tasklet_setup(&ctx->bh, cdc_ncm_txpath_bh);
        atomic_set(&ctx->stop, 0);
        spin_lock_init(&ctx->mtx);

        /* store ctx pointer in device data field */
        dev->data[0] = (unsigned long)ctx;

        /* only the control interface can be successfully probed */
        ctx->control = intf;

        /* get some pointers */
        driver = driver_of(intf);
        buf = intf->cur_altsetting->extra;
        len = intf->cur_altsetting->extralen;

        /* parse through descriptors associated with control interface */
        cdc_parse_cdc_header(&hdr, intf, buf, len);

        if (hdr.usb_cdc_union_desc)
                ctx->data = usb_ifnum_to_if(dev->udev,
                                            hdr.usb_cdc_union_desc->bSlaveInterface0);
        ctx->ether_desc = hdr.usb_cdc_ether_desc;
        ctx->func_desc = hdr.usb_cdc_ncm_desc;
        ctx->mbim_desc = hdr.usb_cdc_mbim_desc;
        ctx->mbim_extended_desc = hdr.usb_cdc_mbim_extended_desc;

        /* some buggy devices have an IAD but no CDC Union */
        if (!hdr.usb_cdc_union_desc && intf->intf_assoc && intf->intf_assoc->bInterfaceCount == 2) {
                ctx->data = usb_ifnum_to_if(dev->udev, intf->cur_altsetting->desc.bInterfaceNumber + 1);
                dev_dbg(&intf->dev, "CDC Union missing - got slave from IAD\n");
        }

        /* check if we got everything */
        if (!ctx->data) {
                dev_err(&intf->dev, "CDC Union missing and no IAD found\n");
                goto error;
        }
        if (cdc_ncm_comm_intf_is_mbim(intf->cur_altsetting)) {
                if (!ctx->mbim_desc) {
                        dev_err(&intf->dev, "MBIM functional descriptor missing\n");
                        goto error;
                }
        } else {
                if (!ctx->ether_desc || !ctx->func_desc) {
                        dev_err(&intf->dev, "NCM or ECM functional descriptors missing\n");
                        goto error;
                }
        }

        /* claim data interface, if different from control */
        if (ctx->data != ctx->control) {
                temp = usb_driver_claim_interface(driver, ctx->data, dev);
                if (temp) {
                        dev_err(&intf->dev, "failed to claim data intf\n");
                        goto error;
                }
        }

        if (ctx->func_desc)
                ctx->filtering_supported = !!(ctx->func_desc->bmNetworkCapabilities
                        & USB_CDC_NCM_NCAP_ETH_FILTER);

        iface_no = ctx->data->cur_altsetting->desc.bInterfaceNumber;

        /* Device-specific flags */
        ctx->drvflags = drvflags;

        /* Reset data interface. Some devices will not reset properly
         * unless they are configured first.  Toggle the altsetting to
         * force a reset.
         * Some other devices do not work properly with this procedure
         * that can be avoided using quirk CDC_MBIM_FLAG_AVOID_ALTSETTING_TOGGLE
         */
        if (!(ctx->drvflags & CDC_MBIM_FLAG_AVOID_ALTSETTING_TOGGLE))
                usb_set_interface(dev->udev, iface_no, data_altsetting);

        temp = usb_set_interface(dev->udev, iface_no, 0);
        if (temp) {
                dev_dbg(&intf->dev, "set interface failed\n");
                goto error2;
        }

        /* initialize basic device settings */
        if (cdc_ncm_init(dev))
                goto error2;

        /* Some firmwares need a pause here or they will silently fail
         * to set up the interface properly.  This value was decided
         * empirically on a Sierra Wireless MC7455 running 02.08.02.00
         * firmware.
         */
        usleep_range(10000, 20000);

        /* configure data interface */
        temp = usb_set_interface(dev->udev, iface_no, data_altsetting);
        if (temp) {
                dev_dbg(&intf->dev, "set interface failed\n");
                goto error2;
        }

        cdc_ncm_find_endpoints(dev, ctx->data);
        cdc_ncm_find_endpoints(dev, ctx->control);
        if (!dev->in || !dev->out ||
            (!dev->status && dev->driver_info->flags & FLAG_LINK_INTR)) {
                dev_dbg(&intf->dev, "failed to collect endpoints\n");
                goto error2;
        }

        usb_set_intfdata(ctx->control, dev);

        if (ctx->ether_desc) {
                temp = usbnet_get_ethernet_addr(dev, ctx->ether_desc->iMACAddress);
                if (temp) {
                        dev_err(&intf->dev, "failed to get mac address\n");
                        goto error2;
                }
                dev_info(&intf->dev, "MAC-Address: %pM\n", dev->net->dev_addr);
        }

        /* finish setting up the device specific data */
        cdc_ncm_setup(dev);

        /* Allocate the delayed NDP if needed. */
        if (ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END) {
                if (ctx->is_ndp16) {
                        ctx->delayed_ndp16 = kzalloc(ctx->max_ndp_size, GFP_KERNEL);
                        if (!ctx->delayed_ndp16)
                                goto error2;
                } else {
                        ctx->delayed_ndp32 = kzalloc(ctx->max_ndp_size, GFP_KERNEL);
                        if (!ctx->delayed_ndp32)
                                goto error2;
                }
                dev_info(&intf->dev, "NDP will be placed at end of frame for this device.");
        }

        /* override ethtool_ops */
        dev->net->ethtool_ops = &cdc_ncm_ethtool_ops;

        /* add our sysfs attrs */
        dev->net->sysfs_groups[0] = &cdc_ncm_sysfs_attr_group;

        /* must handle MTU changes */
        dev->net->netdev_ops = &cdc_ncm_netdev_ops;
        dev->net->max_mtu = cdc_ncm_max_dgram_size(dev) - cdc_ncm_eth_hlen(dev);

        return 0;

error2:
        usb_set_intfdata(ctx->control, NULL);
        usb_set_intfdata(ctx->data, NULL);
        if (ctx->data != ctx->control)
                usb_driver_release_interface(driver, ctx->data);
error:
        cdc_ncm_free((struct cdc_ncm_ctx *)dev->data[0]);
        dev->data[0] = 0;
        dev_info(&intf->dev, "bind() failure\n");
        return -ENODEV;
}
EXPORT_SYMBOL_GPL(cdc_ncm_bind_common);

void cdc_ncm_unbind(struct usbnet *dev, struct usb_interface *intf)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        struct usb_driver *driver = driver_of(intf);

        if (ctx == NULL)
                return;                /* no setup */

        atomic_set(&ctx->stop, 1);

        hrtimer_cancel(&ctx->tx_timer);

        tasklet_kill(&ctx->bh);

        /* handle devices with combined control and data interface */
        if (ctx->control == ctx->data)
                ctx->data = NULL;

        /* disconnect master --> disconnect slave */
        if (intf == ctx->control && ctx->data) {
                usb_set_intfdata(ctx->data, NULL);
                usb_driver_release_interface(driver, ctx->data);
                ctx->data = NULL;

        } else if (intf == ctx->data && ctx->control) {
                usb_set_intfdata(ctx->control, NULL);
                usb_driver_release_interface(driver, ctx->control);
                ctx->control = NULL;
        }

        usb_set_intfdata(intf, NULL);
        cdc_ncm_free(ctx);
}
EXPORT_SYMBOL_GPL(cdc_ncm_unbind);

/* Return the number of the MBIM control interface altsetting iff it
 * is preferred and available,
 */
u8 cdc_ncm_select_altsetting(struct usb_interface *intf)
{
        struct usb_host_interface *alt;

        /* The MBIM spec defines a NCM compatible default altsetting,
         * which we may have matched:
         *
         *  "Functions that implement both NCM 1.0 and MBIM (an
         *   “NCM/MBIM function”) according to this recommendation
         *   shall provide two alternate settings for the
         *   Communication Interface.  Alternate setting 0, and the
         *   associated class and endpoint descriptors, shall be
         *   constructed according to the rules given for the
         *   Communication Interface in section 5 of [USBNCM10].
         *   Alternate setting 1, and the associated class and
         *   endpoint descriptors, shall be constructed according to
         *   the rules given in section 6 (USB Device Model) of this
         *   specification."
         */
        if (intf->num_altsetting < 2)
                return intf->cur_altsetting->desc.bAlternateSetting;

        if (prefer_mbim) {
                alt = usb_altnum_to_altsetting(intf, CDC_NCM_COMM_ALTSETTING_MBIM);
                if (alt && cdc_ncm_comm_intf_is_mbim(alt))
                        return CDC_NCM_COMM_ALTSETTING_MBIM;
        }
        return CDC_NCM_COMM_ALTSETTING_NCM;
}
EXPORT_SYMBOL_GPL(cdc_ncm_select_altsetting);

static int cdc_ncm_bind(struct usbnet *dev, struct usb_interface *intf)
{
        /* MBIM backwards compatible function? */
        if (cdc_ncm_select_altsetting(intf) != CDC_NCM_COMM_ALTSETTING_NCM)
                return -ENODEV;

        /* The NCM data altsetting is fixed, so we hard-coded it.
         * Additionally, generic NCM devices are assumed to accept arbitrarily
         * placed NDP.
         */
        return cdc_ncm_bind_common(dev, intf, CDC_NCM_DATA_ALTSETTING_NCM, 0);
}

static void cdc_ncm_align_tail(struct sk_buff *skb, size_t modulus, size_t remainder, size_t max)
{
        size_t align = ALIGN(skb->len, modulus) - skb->len + remainder;

        if (skb->len + align > max)
                align = max - skb->len;
        if (align && skb_tailroom(skb) >= align)
                skb_put_zero(skb, align);
}

/* return a pointer to a valid struct usb_cdc_ncm_ndp16 of type sign, possibly
 * allocating a new one within skb
 */
static struct usb_cdc_ncm_ndp16 *cdc_ncm_ndp16(struct cdc_ncm_ctx *ctx, struct sk_buff *skb, __le32 sign, size_t reserve)
{
        struct usb_cdc_ncm_ndp16 *ndp16 = NULL;
        struct usb_cdc_ncm_nth16 *nth16 = (void *)skb->data;
        size_t ndpoffset = le16_to_cpu(nth16->wNdpIndex);

        /* If NDP should be moved to the end of the NCM package, we can't follow the
        * NTH16 header as we would normally do. NDP isn't written to the SKB yet, and
        * the wNdpIndex field in the header is actually not consistent with reality. It will be later.
        */
        if (ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END) {
                if (ctx->delayed_ndp16->dwSignature == sign)
                        return ctx->delayed_ndp16;

                /* We can only push a single NDP to the end. Return
                 * NULL to send what we've already got and queue this
                 * skb for later.
                 */
                else if (ctx->delayed_ndp16->dwSignature)
                        return NULL;
        }

        /* follow the chain of NDPs, looking for a match */
        while (ndpoffset) {
                ndp16 = (struct usb_cdc_ncm_ndp16 *)(skb->data + ndpoffset);
                if  (ndp16->dwSignature == sign)
                        return ndp16;
                ndpoffset = le16_to_cpu(ndp16->wNextNdpIndex);
        }

        /* align new NDP */
        if (!(ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END))
                cdc_ncm_align_tail(skb, ctx->tx_ndp_modulus, 0, ctx->tx_curr_size);

        /* verify that there is room for the NDP and the datagram (reserve) */
        if ((ctx->tx_curr_size - skb->len - reserve) < ctx->max_ndp_size)
                return NULL;

        /* link to it */
        if (ndp16)
                ndp16->wNextNdpIndex = cpu_to_le16(skb->len);
        else
                nth16->wNdpIndex = cpu_to_le16(skb->len);

        /* push a new empty NDP */
        if (!(ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END))
                ndp16 = skb_put_zero(skb, ctx->max_ndp_size);
        else
                ndp16 = ctx->delayed_ndp16;

        ndp16->dwSignature = sign;
        ndp16->wLength = cpu_to_le16(sizeof(struct usb_cdc_ncm_ndp16) + sizeof(struct usb_cdc_ncm_dpe16));
        return ndp16;
}

static struct usb_cdc_ncm_ndp32 *cdc_ncm_ndp32(struct cdc_ncm_ctx *ctx, struct sk_buff *skb, __le32 sign, size_t reserve)
{
        struct usb_cdc_ncm_ndp32 *ndp32 = NULL;
        struct usb_cdc_ncm_nth32 *nth32 = (void *)skb->data;
        size_t ndpoffset = le32_to_cpu(nth32->dwNdpIndex);

        /* If NDP should be moved to the end of the NCM package, we can't follow the
         * NTH32 header as we would normally do. NDP isn't written to the SKB yet, and
         * the wNdpIndex field in the header is actually not consistent with reality. It will be later.
         */
        if (ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END) {
                if (ctx->delayed_ndp32->dwSignature == sign)
                        return ctx->delayed_ndp32;

                /* We can only push a single NDP to the end. Return
                 * NULL to send what we've already got and queue this
                 * skb for later.
                 */
                else if (ctx->delayed_ndp32->dwSignature)
                        return NULL;
        }

        /* follow the chain of NDPs, looking for a match */
        while (ndpoffset) {
                ndp32 = (struct usb_cdc_ncm_ndp32 *)(skb->data + ndpoffset);
                if  (ndp32->dwSignature == sign)
                        return ndp32;
                ndpoffset = le32_to_cpu(ndp32->dwNextNdpIndex);
        }

        /* align new NDP */
        if (!(ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END))
                cdc_ncm_align_tail(skb, ctx->tx_ndp_modulus, 0, ctx->tx_curr_size);

        /* verify that there is room for the NDP and the datagram (reserve) */
        if ((ctx->tx_curr_size - skb->len - reserve) < ctx->max_ndp_size)
                return NULL;

        /* link to it */
        if (ndp32)
                ndp32->dwNextNdpIndex = cpu_to_le32(skb->len);
        else
                nth32->dwNdpIndex = cpu_to_le32(skb->len);

        /* push a new empty NDP */
        if (!(ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END))
                ndp32 = skb_put_zero(skb, ctx->max_ndp_size);
        else
                ndp32 = ctx->delayed_ndp32;

        ndp32->dwSignature = sign;
        ndp32->wLength = cpu_to_le16(sizeof(struct usb_cdc_ncm_ndp32) + sizeof(struct usb_cdc_ncm_dpe32));
        return ndp32;
}

struct sk_buff *
cdc_ncm_fill_tx_frame(struct usbnet *dev, struct sk_buff *skb, __le32 sign)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        union {
                struct usb_cdc_ncm_nth16 *nth16;
                struct usb_cdc_ncm_nth32 *nth32;
        } nth;
        union {
                struct usb_cdc_ncm_ndp16 *ndp16;
                struct usb_cdc_ncm_ndp32 *ndp32;
        } ndp;
        struct sk_buff *skb_out;
        u16 n = 0, index, ndplen;
        u8 ready2send = 0;
        u32 delayed_ndp_size;
        size_t padding_count;

        /* When our NDP gets written in cdc_ncm_ndp(), then skb_out->len gets updated
         * accordingly. Otherwise, we should check here.
         */
        if (ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END)
                delayed_ndp_size = ctx->max_ndp_size +
                        max_t(u32,
                              ctx->tx_ndp_modulus,
                              ctx->tx_modulus + ctx->tx_remainder) - 1;
        else
                delayed_ndp_size = 0;

        /* if there is a remaining skb, it gets priority */
        if (skb != NULL) {
                swap(skb, ctx->tx_rem_skb);
                swap(sign, ctx->tx_rem_sign);
        } else {
                ready2send = 1;
        }

        /* check if we are resuming an OUT skb */
        skb_out = ctx->tx_curr_skb;

        /* allocate a new OUT skb */
        if (!skb_out) {
                if (ctx->tx_low_mem_val == 0) {
                        ctx->tx_curr_size = ctx->tx_max;
                        skb_out = alloc_skb(ctx->tx_curr_size, GFP_ATOMIC);
                        /* If the memory allocation fails we will wait longer
                         * each time before attempting another full size
                         * allocation again to not overload the system
                         * further.
                         */
                        if (skb_out == NULL) {
                                /* If even the smallest allocation fails, abort. */
                                if (ctx->tx_curr_size == USB_CDC_NCM_NTB_MIN_OUT_SIZE)
                                        goto alloc_failed;
                                ctx->tx_low_mem_max_cnt = min(ctx->tx_low_mem_max_cnt + 1,
                                                              (unsigned)CDC_NCM_LOW_MEM_MAX_CNT);
                                ctx->tx_low_mem_val = ctx->tx_low_mem_max_cnt;
                        }
                }
                if (skb_out == NULL) {
                        /* See if a very small allocation is possible.
                         * We will send this packet immediately and hope
                         * that there is more memory available later.
                         */
                        if (skb)
                                ctx->tx_curr_size = max(skb->len,
                                        (u32)USB_CDC_NCM_NTB_MIN_OUT_SIZE);
                        else
                                ctx->tx_curr_size = USB_CDC_NCM_NTB_MIN_OUT_SIZE;
                        skb_out = alloc_skb(ctx->tx_curr_size, GFP_ATOMIC);

                        /* No allocation possible so we will abort */
                        if (!skb_out)
                                goto alloc_failed;
                        ctx->tx_low_mem_val--;
                }
                if (ctx->is_ndp16) {
                        /* fill out the initial 16-bit NTB header */
                        nth.nth16 = skb_put_zero(skb_out, sizeof(struct usb_cdc_ncm_nth16));
                        nth.nth16->dwSignature = cpu_to_le32(USB_CDC_NCM_NTH16_SIGN);
                        nth.nth16->wHeaderLength = cpu_to_le16(sizeof(struct usb_cdc_ncm_nth16));
                        nth.nth16->wSequence = cpu_to_le16(ctx->tx_seq++);
                } else {
                        /* fill out the initial 32-bit NTB header */
                        nth.nth32 = skb_put_zero(skb_out, sizeof(struct usb_cdc_ncm_nth32));
                        nth.nth32->dwSignature = cpu_to_le32(USB_CDC_NCM_NTH32_SIGN);
                        nth.nth32->wHeaderLength = cpu_to_le16(sizeof(struct usb_cdc_ncm_nth32));
                        nth.nth32->wSequence = cpu_to_le16(ctx->tx_seq++);
                }

                /* count total number of frames in this NTB */
                ctx->tx_curr_frame_num = 0;

                /* recent payload counter for this skb_out */
                ctx->tx_curr_frame_payload = 0;
        }

        for (n = ctx->tx_curr_frame_num; n < ctx->tx_max_datagrams; n++) {
                /* send any remaining skb first */
                if (skb == NULL) {
                        skb = ctx->tx_rem_skb;
                        sign = ctx->tx_rem_sign;
                        ctx->tx_rem_skb = NULL;

                        /* check for end of skb */
                        if (skb == NULL)
                                break;
                }

                /* get the appropriate NDP for this skb */
                if (ctx->is_ndp16)
                        ndp.ndp16 = cdc_ncm_ndp16(ctx, skb_out, sign, skb->len + ctx->tx_modulus + ctx->tx_remainder);
                else
                        ndp.ndp32 = cdc_ncm_ndp32(ctx, skb_out, sign, skb->len + ctx->tx_modulus + ctx->tx_remainder);

                /* align beginning of next frame */
                cdc_ncm_align_tail(skb_out,  ctx->tx_modulus, ctx->tx_remainder, ctx->tx_curr_size);

                /* check if we had enough room left for both NDP and frame */
                if ((ctx->is_ndp16 && !ndp.ndp16) || (!ctx->is_ndp16 && !ndp.ndp32) ||
                    skb_out->len + skb->len + delayed_ndp_size > ctx->tx_curr_size) {
                        if (n == 0) {
                                /* won't fit, MTU problem? */
                                dev_kfree_skb_any(skb);
                                skb = NULL;
                                dev->net->stats.tx_dropped++;
                        } else {
                                /* no room for skb - store for later */
                                if (ctx->tx_rem_skb != NULL) {
                                        dev_kfree_skb_any(ctx->tx_rem_skb);
                                        dev->net->stats.tx_dropped++;
                                }
                                ctx->tx_rem_skb = skb;
                                ctx->tx_rem_sign = sign;
                                skb = NULL;
                                ready2send = 1;
                                ctx->tx_reason_ntb_full++;        /* count reason for transmitting */
                        }
                        break;
                }

                /* calculate frame number within this NDP */
                if (ctx->is_ndp16) {
                        ndplen = le16_to_cpu(ndp.ndp16->wLength);
                        index = (ndplen - sizeof(struct usb_cdc_ncm_ndp16)) / sizeof(struct usb_cdc_ncm_dpe16) - 1;

                        /* OK, add this skb */
                        ndp.ndp16->dpe16[index].wDatagramLength = cpu_to_le16(skb->len);
                        ndp.ndp16->dpe16[index].wDatagramIndex = cpu_to_le16(skb_out->len);
                        ndp.ndp16->wLength = cpu_to_le16(ndplen + sizeof(struct usb_cdc_ncm_dpe16));
                } else {
                        ndplen = le16_to_cpu(ndp.ndp32->wLength);
                        index = (ndplen - sizeof(struct usb_cdc_ncm_ndp32)) / sizeof(struct usb_cdc_ncm_dpe32) - 1;

                        ndp.ndp32->dpe32[index].dwDatagramLength = cpu_to_le32(skb->len);
                        ndp.ndp32->dpe32[index].dwDatagramIndex = cpu_to_le32(skb_out->len);
                        ndp.ndp32->wLength = cpu_to_le16(ndplen + sizeof(struct usb_cdc_ncm_dpe32));
                }
                skb_put_data(skb_out, skb->data, skb->len);
                ctx->tx_curr_frame_payload += skb->len;        /* count real tx payload data */
                dev_kfree_skb_any(skb);
                skb = NULL;

                /* send now if this NDP is full */
                if (index >= CDC_NCM_DPT_DATAGRAMS_MAX) {
                        ready2send = 1;
                        ctx->tx_reason_ndp_full++;        /* count reason for transmitting */
                        break;
                }
        }

        /* free up any dangling skb */
        if (skb != NULL) {
                dev_kfree_skb_any(skb);
                skb = NULL;
                dev->net->stats.tx_dropped++;
        }

        ctx->tx_curr_frame_num = n;

        if (n == 0) {
                /* wait for more frames */
                /* push variables */
                ctx->tx_curr_skb = skb_out;
                goto exit_no_skb;

        } else if ((n < ctx->tx_max_datagrams) && (ready2send == 0) && (ctx->timer_interval > 0)) {
                /* wait for more frames */
                /* push variables */
                ctx->tx_curr_skb = skb_out;
                /* set the pending count */
                if (n < CDC_NCM_RESTART_TIMER_DATAGRAM_CNT)
                        ctx->tx_timer_pending = CDC_NCM_TIMER_PENDING_CNT;
                goto exit_no_skb;

        } else {
                if (n == ctx->tx_max_datagrams)
                        ctx->tx_reason_max_datagram++;        /* count reason for transmitting */
                /* frame goes out */
                /* variables will be reset at next call */
        }

        /* If requested, put NDP at end of frame. */
        if (ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END) {
                if (ctx->is_ndp16) {
                        nth.nth16 = (struct usb_cdc_ncm_nth16 *)skb_out->data;
                        cdc_ncm_align_tail(skb_out, ctx->tx_ndp_modulus, 0, ctx->tx_curr_size - ctx->max_ndp_size);
                        nth.nth16->wNdpIndex = cpu_to_le16(skb_out->len);
                        skb_put_data(skb_out, ctx->delayed_ndp16, ctx->max_ndp_size);

                        /* Zero out delayed NDP - signature checking will naturally fail. */
                        ndp.ndp16 = memset(ctx->delayed_ndp16, 0, ctx->max_ndp_size);
                } else {
                        nth.nth32 = (struct usb_cdc_ncm_nth32 *)skb_out->data;
                        cdc_ncm_align_tail(skb_out, ctx->tx_ndp_modulus, 0, ctx->tx_curr_size - ctx->max_ndp_size);
                        nth.nth32->dwNdpIndex = cpu_to_le32(skb_out->len);
                        skb_put_data(skb_out, ctx->delayed_ndp32, ctx->max_ndp_size);

                        ndp.ndp32 = memset(ctx->delayed_ndp32, 0, ctx->max_ndp_size);
                }
        }

        /* If collected data size is less or equal ctx->min_tx_pkt
         * bytes, we send buffers as it is. If we get more data, it
         * would be more efficient for USB HS mobile device with DMA
         * engine to receive a full size NTB, than canceling DMA
         * transfer and receiving a short packet.
         *
         * This optimization support is pointless if we end up sending
         * a ZLP after full sized NTBs.
         */
        if (!(dev->driver_info->flags & FLAG_SEND_ZLP) &&
            skb_out->len > ctx->min_tx_pkt) {
                padding_count = ctx->tx_curr_size - skb_out->len;
                if (!WARN_ON(padding_count > ctx->tx_curr_size))
                        skb_put_zero(skb_out, padding_count);
        } else if (skb_out->len < ctx->tx_curr_size &&
                   (skb_out->len % dev->maxpacket) == 0) {
                skb_put_u8(skb_out, 0);        /* force short packet */
        }

        /* set final frame length */
        if (ctx->is_ndp16) {
                nth.nth16 = (struct usb_cdc_ncm_nth16 *)skb_out->data;
                nth.nth16->wBlockLength = cpu_to_le16(skb_out->len);
        } else {
                nth.nth32 = (struct usb_cdc_ncm_nth32 *)skb_out->data;
                nth.nth32->dwBlockLength = cpu_to_le32(skb_out->len);
        }

        /* return skb */
        ctx->tx_curr_skb = NULL;

        /* keep private stats: framing overhead and number of NTBs */
        ctx->tx_overhead += skb_out->len - ctx->tx_curr_frame_payload;
        ctx->tx_ntbs++;

        /* usbnet will count all the framing overhead by default.
         * Adjust the stats so that the tx_bytes counter show real
         * payload data instead.
         */
        usbnet_set_skb_tx_stats(skb_out, n,
                                (long)ctx->tx_curr_frame_payload - skb_out->len);

        return skb_out;

alloc_failed:
        if (skb) {
                dev_kfree_skb_any(skb);
                dev->net->stats.tx_dropped++;
        }
exit_no_skb:
        /* Start timer, if there is a remaining non-empty skb */
        if (ctx->tx_curr_skb != NULL && n > 0)
                cdc_ncm_tx_timeout_start(ctx);
        return NULL;
}
EXPORT_SYMBOL_GPL(cdc_ncm_fill_tx_frame);

static void cdc_ncm_tx_timeout_start(struct cdc_ncm_ctx *ctx)
{
        /* start timer, if not already started */
        if (!(hrtimer_active(&ctx->tx_timer) || atomic_read(&ctx->stop)))
                hrtimer_start(&ctx->tx_timer,
                                ctx->timer_interval,
                                HRTIMER_MODE_REL);
}

static enum hrtimer_restart cdc_ncm_tx_timer_cb(struct hrtimer *timer)
{
        struct cdc_ncm_ctx *ctx =
                        container_of(timer, struct cdc_ncm_ctx, tx_timer);

        if (!atomic_read(&ctx->stop))
                tasklet_schedule(&ctx->bh);
        return HRTIMER_NORESTART;
}

static void cdc_ncm_txpath_bh(struct tasklet_struct *t)
{
        struct cdc_ncm_ctx *ctx = from_tasklet(ctx, t, bh);
        struct usbnet *dev = ctx->dev;

        spin_lock(&ctx->mtx);
        if (ctx->tx_timer_pending != 0) {
                ctx->tx_timer_pending--;
                cdc_ncm_tx_timeout_start(ctx);
                spin_unlock(&ctx->mtx);
        } else if (dev->net != NULL) {
                ctx->tx_reason_timeout++;        /* count reason for transmitting */
                spin_unlock(&ctx->mtx);
                netif_tx_lock_bh(dev->net);
                usbnet_start_xmit(NULL, dev->net);
                netif_tx_unlock_bh(dev->net);
        } else {
                spin_unlock(&ctx->mtx);
        }
}

struct sk_buff *
cdc_ncm_tx_fixup(struct usbnet *dev, struct sk_buff *skb, gfp_t flags)
{
        struct sk_buff *skb_out;
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];

        /*
         * The Ethernet API we are using does not support transmitting
         * multiple Ethernet frames in a single call. This driver will
         * accumulate multiple Ethernet frames and send out a larger
         * USB frame when the USB buffer is full or when a single jiffies
         * timeout happens.
         */
        if (ctx == NULL)
                goto error;

        spin_lock_bh(&ctx->mtx);

        if (ctx->is_ndp16)
                skb_out = cdc_ncm_fill_tx_frame(dev, skb, cpu_to_le32(USB_CDC_NCM_NDP16_NOCRC_SIGN));
        else
                skb_out = cdc_ncm_fill_tx_frame(dev, skb, cpu_to_le32(USB_CDC_NCM_NDP32_NOCRC_SIGN));

        spin_unlock_bh(&ctx->mtx);
        return skb_out;

error:
        if (skb != NULL)
                dev_kfree_skb_any(skb);

        return NULL;
}
EXPORT_SYMBOL_GPL(cdc_ncm_tx_fixup);

/* verify NTB header and return offset of first NDP, or negative error */
int cdc_ncm_rx_verify_nth16(struct cdc_ncm_ctx *ctx, struct sk_buff *skb_in)
{
        struct usbnet *dev = netdev_priv(skb_in->dev);
        struct usb_cdc_ncm_nth16 *nth16;
        int len;
        int ret = -EINVAL;

        if (ctx == NULL)
                goto error;

        if (skb_in->len < (sizeof(struct usb_cdc_ncm_nth16) +
                                        sizeof(struct usb_cdc_ncm_ndp16))) {
                netif_dbg(dev, rx_err, dev->net, "frame too short\n");
                goto error;
        }

        nth16 = (struct usb_cdc_ncm_nth16 *)skb_in->data;

        if (nth16->dwSignature != cpu_to_le32(USB_CDC_NCM_NTH16_SIGN)) {
                netif_dbg(dev, rx_err, dev->net,
                          "invalid NTH16 signature <%#010x>\n",
                          le32_to_cpu(nth16->dwSignature));
                goto error;
        }

        len = le16_to_cpu(nth16->wBlockLength);
        if (len > ctx->rx_max) {
                netif_dbg(dev, rx_err, dev->net,
                          "unsupported NTB block length %u/%u\n", len,
                          ctx->rx_max);
                goto error;
        }

        if ((ctx->rx_seq + 1) != le16_to_cpu(nth16->wSequence) &&
            (ctx->rx_seq || le16_to_cpu(nth16->wSequence)) &&
            !((ctx->rx_seq == 0xffff) && !le16_to_cpu(nth16->wSequence))) {
                netif_dbg(dev, rx_err, dev->net,
                          "sequence number glitch prev=%d curr=%d\n",
                          ctx->rx_seq, le16_to_cpu(nth16->wSequence));
        }
        ctx->rx_seq = le16_to_cpu(nth16->wSequence);

        ret = le16_to_cpu(nth16->wNdpIndex);
error:
        return ret;
}
EXPORT_SYMBOL_GPL(cdc_ncm_rx_verify_nth16);

int cdc_ncm_rx_verify_nth32(struct cdc_ncm_ctx *ctx, struct sk_buff *skb_in)
{
        struct usbnet *dev = netdev_priv(skb_in->dev);
        struct usb_cdc_ncm_nth32 *nth32;
        int len;
        int ret = -EINVAL;

        if (ctx == NULL)
                goto error;

        if (skb_in->len < (sizeof(struct usb_cdc_ncm_nth32) +
                                        sizeof(struct usb_cdc_ncm_ndp32))) {
                netif_dbg(dev, rx_err, dev->net, "frame too short\n");
                goto error;
        }

        nth32 = (struct usb_cdc_ncm_nth32 *)skb_in->data;

        if (nth32->dwSignature != cpu_to_le32(USB_CDC_NCM_NTH32_SIGN)) {
                netif_dbg(dev, rx_err, dev->net,
                          "invalid NTH32 signature <%#010x>\n",
                          le32_to_cpu(nth32->dwSignature));
                goto error;
        }

        len = le32_to_cpu(nth32->dwBlockLength);
        if (len > ctx->rx_max) {
                netif_dbg(dev, rx_err, dev->net,
                          "unsupported NTB block length %u/%u\n", len,
                          ctx->rx_max);
                goto error;
        }

        if ((ctx->rx_seq + 1) != le16_to_cpu(nth32->wSequence) &&
            (ctx->rx_seq || le16_to_cpu(nth32->wSequence)) &&
            !((ctx->rx_seq == 0xffff) && !le16_to_cpu(nth32->wSequence))) {
                netif_dbg(dev, rx_err, dev->net,
                          "sequence number glitch prev=%d curr=%d\n",
                          ctx->rx_seq, le16_to_cpu(nth32->wSequence));
        }
        ctx->rx_seq = le16_to_cpu(nth32->wSequence);

        ret = le32_to_cpu(nth32->dwNdpIndex);
error:
        return ret;
}
EXPORT_SYMBOL_GPL(cdc_ncm_rx_verify_nth32);

/* verify NDP header and return number of datagrams, or negative error */
int cdc_ncm_rx_verify_ndp16(struct sk_buff *skb_in, int ndpoffset)
{
        struct usbnet *dev = netdev_priv(skb_in->dev);
        struct usb_cdc_ncm_ndp16 *ndp16;
        int ret = -EINVAL;

        if ((ndpoffset + sizeof(struct usb_cdc_ncm_ndp16)) > skb_in->len) {
                netif_dbg(dev, rx_err, dev->net, "invalid NDP offset  <%u>\n",
                          ndpoffset);
                goto error;
        }
        ndp16 = (struct usb_cdc_ncm_ndp16 *)(skb_in->data + ndpoffset);

        if (le16_to_cpu(ndp16->wLength) < USB_CDC_NCM_NDP16_LENGTH_MIN) {
                netif_dbg(dev, rx_err, dev->net, "invalid DPT16 length <%u>\n",
                          le16_to_cpu(ndp16->wLength));
                goto error;
        }

        ret = ((le16_to_cpu(ndp16->wLength) -
                                        sizeof(struct usb_cdc_ncm_ndp16)) /
                                        sizeof(struct usb_cdc_ncm_dpe16));
        ret--; /* we process NDP entries except for the last one */

        if ((sizeof(struct usb_cdc_ncm_ndp16) +
             ret * (sizeof(struct usb_cdc_ncm_dpe16))) > skb_in->len) {
                netif_dbg(dev, rx_err, dev->net, "Invalid nframes = %d\n", ret);
                ret = -EINVAL;
        }

error:
        return ret;
}
EXPORT_SYMBOL_GPL(cdc_ncm_rx_verify_ndp16);

/* verify NDP header and return number of datagrams, or negative error */
int cdc_ncm_rx_verify_ndp32(struct sk_buff *skb_in, int ndpoffset)
{
        struct usbnet *dev = netdev_priv(skb_in->dev);
        struct usb_cdc_ncm_ndp32 *ndp32;
        int ret = -EINVAL;

        if ((ndpoffset + sizeof(struct usb_cdc_ncm_ndp32)) > skb_in->len) {
                netif_dbg(dev, rx_err, dev->net, "invalid NDP offset  <%u>\n",
                          ndpoffset);
                goto error;
        }
        ndp32 = (struct usb_cdc_ncm_ndp32 *)(skb_in->data + ndpoffset);

        if (le16_to_cpu(ndp32->wLength) < USB_CDC_NCM_NDP32_LENGTH_MIN) {
                netif_dbg(dev, rx_err, dev->net, "invalid DPT32 length <%u>\n",
                          le16_to_cpu(ndp32->wLength));
                goto error;
        }

        ret = ((le16_to_cpu(ndp32->wLength) -
                                        sizeof(struct usb_cdc_ncm_ndp32)) /
                                        sizeof(struct usb_cdc_ncm_dpe32));
        ret--; /* we process NDP entries except for the last one */

        if ((sizeof(struct usb_cdc_ncm_ndp32) +
             ret * (sizeof(struct usb_cdc_ncm_dpe32))) > skb_in->len) {
                netif_dbg(dev, rx_err, dev->net, "Invalid nframes = %d\n", ret);
                ret = -EINVAL;
        }

error:
        return ret;
}
EXPORT_SYMBOL_GPL(cdc_ncm_rx_verify_ndp32);

int cdc_ncm_rx_fixup(struct usbnet *dev, struct sk_buff *skb_in)
{
        struct sk_buff *skb;
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
        unsigned int len;
        int nframes;
        int x;
        unsigned int offset;
        union {
                struct usb_cdc_ncm_ndp16 *ndp16;
                struct usb_cdc_ncm_ndp32 *ndp32;
        } ndp;
        union {
                struct usb_cdc_ncm_dpe16 *dpe16;
                struct usb_cdc_ncm_dpe32 *dpe32;
        } dpe;

        int ndpoffset;
        int loopcount = 50; /* arbitrary max preventing infinite loop */
        u32 payload = 0;

        if (ctx->is_ndp16)
                ndpoffset = cdc_ncm_rx_verify_nth16(ctx, skb_in);
        else
                ndpoffset = cdc_ncm_rx_verify_nth32(ctx, skb_in);

        if (ndpoffset < 0)
                goto error;

next_ndp:
        if (ctx->is_ndp16) {
                nframes = cdc_ncm_rx_verify_ndp16(skb_in, ndpoffset);
                if (nframes < 0)
                        goto error;

                ndp.ndp16 = (struct usb_cdc_ncm_ndp16 *)(skb_in->data + ndpoffset);

                if (ndp.ndp16->dwSignature != cpu_to_le32(USB_CDC_NCM_NDP16_NOCRC_SIGN)) {
                        netif_dbg(dev, rx_err, dev->net,
                                  "invalid DPT16 signature <%#010x>\n",
                                  le32_to_cpu(ndp.ndp16->dwSignature));
                        goto err_ndp;
                }
                dpe.dpe16 = ndp.ndp16->dpe16;
        } else {
                nframes = cdc_ncm_rx_verify_ndp32(skb_in, ndpoffset);
                if (nframes < 0)
                        goto error;

                ndp.ndp32 = (struct usb_cdc_ncm_ndp32 *)(skb_in->data + ndpoffset);

                if (ndp.ndp32->dwSignature != cpu_to_le32(USB_CDC_NCM_NDP32_NOCRC_SIGN)) {
                        netif_dbg(dev, rx_err, dev->net,
                                  "invalid DPT32 signature <%#010x>\n",
                                  le32_to_cpu(ndp.ndp32->dwSignature));
                        goto err_ndp;
                }
                dpe.dpe32 = ndp.ndp32->dpe32;
        }

        for (x = 0; x < nframes; x++) {
                if (ctx->is_ndp16) {
                        offset = le16_to_cpu(dpe.dpe16->wDatagramIndex);
                        len = le16_to_cpu(dpe.dpe16->wDatagramLength);
                } else {
                        offset = le32_to_cpu(dpe.dpe32->dwDatagramIndex);
                        len = le32_to_cpu(dpe.dpe32->dwDatagramLength);
                }

                /*
                 * CDC NCM ch. 3.7
                 * All entries after first NULL entry are to be ignored
                 */
                if ((offset == 0) || (len == 0)) {
                        if (!x)
                                goto err_ndp; /* empty NTB */
                        break;
                }

                /* sanity checking - watch out for integer wrap*/
                if ((offset > skb_in->len) || (len > skb_in->len - offset) ||
                                (len > ctx->rx_max) || (len < ETH_HLEN)) {
                        netif_dbg(dev, rx_err, dev->net,
                                  "invalid frame detected (ignored) offset[%u]=%u, length=%u, skb=%p\n",
                                  x, offset, len, skb_in);
                        if (!x)
                                goto err_ndp;
                        break;

                } else {
                        /* create a fresh copy to reduce truesize */
                        skb = netdev_alloc_skb_ip_align(dev->net,  len);
                        if (!skb)
                                goto error;
                        skb_put_data(skb, skb_in->data + offset, len);
                        usbnet_skb_return(dev, skb);
                        payload += len;        /* count payload bytes in this NTB */
                }

                if (ctx->is_ndp16)
                        dpe.dpe16++;
                else
                        dpe.dpe32++;
        }
err_ndp:
        /* are there more NDPs to process? */
        if (ctx->is_ndp16)
                ndpoffset = le16_to_cpu(ndp.ndp16->wNextNdpIndex);
        else
                ndpoffset = le32_to_cpu(ndp.ndp32->dwNextNdpIndex);

        if (ndpoffset && loopcount--)
                goto next_ndp;

        /* update stats */
        ctx->rx_overhead += skb_in->len - payload;
        ctx->rx_ntbs++;

        return 1;
error:
        return 0;
}
EXPORT_SYMBOL_GPL(cdc_ncm_rx_fixup);

static void
cdc_ncm_speed_change(struct usbnet *dev,
                     struct usb_cdc_speed_change *data)
{
        /* RTL8156 shipped before 2021 sends notification about every 32ms. */
        dev->rx_speed = le32_to_cpu(data->DLBitRRate);
        dev->tx_speed = le32_to_cpu(data->ULBitRate);
}

static void cdc_ncm_status(struct usbnet *dev, struct urb *urb)
{
        struct usb_cdc_notification *event;

        if (urb->actual_length < sizeof(*event))
                return;

        /* test for split data in 8-byte chunks */
        if (test_and_clear_bit(EVENT_STS_SPLIT, &dev->flags)) {
                cdc_ncm_speed_change(dev,
                      (struct usb_cdc_speed_change *)urb->transfer_buffer);
                return;
        }

        event = urb->transfer_buffer;

        switch (event->bNotificationType) {
        case USB_CDC_NOTIFY_NETWORK_CONNECTION:
                /*
                 * According to the CDC NCM specification ch.7.1
                 * USB_CDC_NOTIFY_NETWORK_CONNECTION notification shall be
                 * sent by device after USB_CDC_NOTIFY_SPEED_CHANGE.
                 */
                /* RTL8156 shipped before 2021 sends notification about
                 * every 32ms. Don't forward notification if state is same.
                 */
                if (netif_carrier_ok(dev->net) != !!event->wValue)
                        usbnet_link_change(dev, !!event->wValue, 0);
                break;

        case USB_CDC_NOTIFY_SPEED_CHANGE:
                if (urb->actual_length < (sizeof(*event) +
                                        sizeof(struct usb_cdc_speed_change)))
                        set_bit(EVENT_STS_SPLIT, &dev->flags);
                else
                        cdc_ncm_speed_change(dev,
                                             (struct usb_cdc_speed_change *)&event[1]);
                break;

        default:
                dev_dbg(&dev->udev->dev,
                        "NCM: unexpected notification 0x%02x!\n",
                        event->bNotificationType);
                break;
        }
}

static void cdc_ncm_update_filter(struct usbnet *dev)
{
        struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];

        if (ctx->filtering_supported)
                usbnet_cdc_update_filter(dev);
}

static const struct driver_info cdc_ncm_info = {
        .description = "CDC NCM (NO ZLP)",
        .flags = FLAG_POINTTOPOINT | FLAG_NO_SETINT | FLAG_MULTI_PACKET
                        | FLAG_LINK_INTR | FLAG_ETHER,
        .bind = cdc_ncm_bind,
        .unbind = cdc_ncm_unbind,
        .manage_power = usbnet_manage_power,
        .status = cdc_ncm_status,
        .rx_fixup = cdc_ncm_rx_fixup,
        .tx_fixup = cdc_ncm_tx_fixup,
        .set_rx_mode = cdc_ncm_update_filter,
};

/* Same as cdc_ncm_info, but with FLAG_SEND_ZLP  */
static const struct driver_info cdc_ncm_zlp_info = {
        .description = "CDC NCM (SEND ZLP)",
        .flags = FLAG_POINTTOPOINT | FLAG_NO_SETINT | FLAG_MULTI_PACKET
                        | FLAG_LINK_INTR | FLAG_ETHER | FLAG_SEND_ZLP,
        .bind = cdc_ncm_bind,
        .unbind = cdc_ncm_unbind,
        .manage_power = usbnet_manage_power,
        .status = cdc_ncm_status,
        .rx_fixup = cdc_ncm_rx_fixup,
        .tx_fixup = cdc_ncm_tx_fixup,
        .set_rx_mode = cdc_ncm_update_filter,
};

/* Same as cdc_ncm_info, but with FLAG_SEND_ZLP */
static const struct driver_info apple_tethering_interface_info = {
        .description = "CDC NCM (Apple Tethering)",
        .flags = FLAG_POINTTOPOINT | FLAG_NO_SETINT | FLAG_MULTI_PACKET
                        | FLAG_LINK_INTR | FLAG_ETHER | FLAG_SEND_ZLP,
        .bind = cdc_ncm_bind,
        .unbind = cdc_ncm_unbind,
        .manage_power = usbnet_manage_power,
        .status = cdc_ncm_status,
        .rx_fixup = cdc_ncm_rx_fixup,
        .tx_fixup = cdc_ncm_tx_fixup,
        .set_rx_mode = usbnet_cdc_update_filter,
};

/* Same as apple_tethering_interface_info, but without FLAG_LINK_INTR */
static const struct driver_info apple_private_interface_info = {
        .description = "CDC NCM (Apple Private)",
        .flags = FLAG_POINTTOPOINT | FLAG_NO_SETINT | FLAG_MULTI_PACKET
                        | FLAG_ETHER | FLAG_SEND_ZLP,
        .bind = cdc_ncm_bind,
        .unbind = cdc_ncm_unbind,
        .manage_power = usbnet_manage_power,
        .status = cdc_ncm_status,
        .rx_fixup = cdc_ncm_rx_fixup,
        .tx_fixup = cdc_ncm_tx_fixup,
        .set_rx_mode = usbnet_cdc_update_filter,
};

/* Same as cdc_ncm_info, but with FLAG_WWAN */
static const struct driver_info wwan_info = {
        .description = "Mobile Broadband Network Device",
        .flags = FLAG_POINTTOPOINT | FLAG_NO_SETINT | FLAG_MULTI_PACKET
                        | FLAG_LINK_INTR | FLAG_WWAN,
        .bind = cdc_ncm_bind,
        .unbind = cdc_ncm_unbind,
        .manage_power = usbnet_manage_power,
        .status = cdc_ncm_status,
        .rx_fixup = cdc_ncm_rx_fixup,
        .tx_fixup = cdc_ncm_tx_fixup,
        .set_rx_mode = cdc_ncm_update_filter,
};

/* Same as wwan_info, but with FLAG_NOARP  */
static const struct driver_info wwan_noarp_info = {
        .description = "Mobile Broadband Network Device (NO ARP)",
        .flags = FLAG_POINTTOPOINT | FLAG_NO_SETINT | FLAG_MULTI_PACKET
                        | FLAG_LINK_INTR | FLAG_WWAN | FLAG_NOARP,
        .bind = cdc_ncm_bind,
        .unbind = cdc_ncm_unbind,
        .manage_power = usbnet_manage_power,
        .status = cdc_ncm_status,
        .rx_fixup = cdc_ncm_rx_fixup,
        .tx_fixup = cdc_ncm_tx_fixup,
        .set_rx_mode = cdc_ncm_update_filter,
};

static const struct usb_device_id cdc_devs[] = {
        /* iPhone */
        { USB_DEVICE_INTERFACE_NUMBER(0x05ac, 0x12a8, 2),
                .driver_info = (unsigned long)&apple_tethering_interface_info,
        },
        { USB_DEVICE_INTERFACE_NUMBER(0x05ac, 0x12a8, 4),
                .driver_info = (unsigned long)&apple_private_interface_info,
        },

        /* iPad */
        { USB_DEVICE_INTERFACE_NUMBER(0x05ac, 0x12ab, 2),
                .driver_info = (unsigned long)&apple_tethering_interface_info,
        },
        { USB_DEVICE_INTERFACE_NUMBER(0x05ac, 0x12ab, 4),
                .driver_info = (unsigned long)&apple_private_interface_info,
        },

        /* Ericsson MBM devices like F5521gw */
        { .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
                | USB_DEVICE_ID_MATCH_VENDOR,
          .idVendor = 0x0bdb,
          .bInterfaceClass = USB_CLASS_COMM,
          .bInterfaceSubClass = USB_CDC_SUBCLASS_NCM,
          .bInterfaceProtocol = USB_CDC_PROTO_NONE,
          .driver_info = (unsigned long) &wwan_info,
        },

        /* Telit LE910 V2 */
        { USB_DEVICE_AND_INTERFACE_INFO(0x1bc7, 0x0036,
                USB_CLASS_COMM,
                USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE),
          .driver_info = (unsigned long)&wwan_noarp_info,
        },

        /* DW5812 LTE Verizon Mobile Broadband Card
         * Unlike DW5550 this device requires FLAG_NOARP
         */
        { USB_DEVICE_AND_INTERFACE_INFO(0x413c, 0x81bb,
                USB_CLASS_COMM,
                USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE),
          .driver_info = (unsigned long)&wwan_noarp_info,
        },

        /* DW5813 LTE AT&T Mobile Broadband Card
         * Unlike DW5550 this device requires FLAG_NOARP
         */
        { USB_DEVICE_AND_INTERFACE_INFO(0x413c, 0x81bc,
                USB_CLASS_COMM,
                USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE),
          .driver_info = (unsigned long)&wwan_noarp_info,
        },

        /* Dell branded MBM devices like DW5550 */
        { .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
                | USB_DEVICE_ID_MATCH_VENDOR,
          .idVendor = 0x413c,
          .bInterfaceClass = USB_CLASS_COMM,
          .bInterfaceSubClass = USB_CDC_SUBCLASS_NCM,
          .bInterfaceProtocol = USB_CDC_PROTO_NONE,
          .driver_info = (unsigned long) &wwan_info,
        },

        /* Toshiba branded MBM devices */
        { .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
                | USB_DEVICE_ID_MATCH_VENDOR,
          .idVendor = 0x0930,
          .bInterfaceClass = USB_CLASS_COMM,
          .bInterfaceSubClass = USB_CDC_SUBCLASS_NCM,
          .bInterfaceProtocol = USB_CDC_PROTO_NONE,
          .driver_info = (unsigned long) &wwan_info,
        },

        /* tag Huawei devices as wwan */
        { USB_VENDOR_AND_INTERFACE_INFO(0x12d1,
                                        USB_CLASS_COMM,
                                        USB_CDC_SUBCLASS_NCM,
                                        USB_CDC_PROTO_NONE),
          .driver_info = (unsigned long)&wwan_info,
        },

        /* Infineon(now Intel) HSPA Modem platform */
        { USB_DEVICE_AND_INTERFACE_INFO(0x1519, 0x0443,
                USB_CLASS_COMM,
                USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE),
          .driver_info = (unsigned long)&wwan_noarp_info,
        },

        /* u-blox TOBY-L4 */
        { USB_DEVICE_AND_INTERFACE_INFO(0x1546, 0x1010,
                USB_CLASS_COMM,
                USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE),
          .driver_info = (unsigned long)&wwan_info,
        },

        /* Intel modem (label from OEM reads Fibocom L850-GL) */
        { USB_DEVICE_AND_INTERFACE_INFO(0x8087, 0x095a,
                USB_CLASS_COMM,
                USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE),
          .driver_info = (unsigned long)&wwan_info,
        },

        /* DisplayLink docking stations */
        { .match_flags = USB_DEVICE_ID_MATCH_INT_INFO
                | USB_DEVICE_ID_MATCH_VENDOR,
          .idVendor = 0x17e9,
          .bInterfaceClass = USB_CLASS_COMM,
          .bInterfaceSubClass = USB_CDC_SUBCLASS_NCM,
          .bInterfaceProtocol = USB_CDC_PROTO_NONE,
          .driver_info = (unsigned long)&cdc_ncm_zlp_info,
        },

        /* Generic CDC-NCM devices */
        { USB_INTERFACE_INFO(USB_CLASS_COMM,
                USB_CDC_SUBCLASS_NCM, USB_CDC_PROTO_NONE),
                .driver_info = (unsigned long)&cdc_ncm_info,
        },
        {
        },
};
MODULE_DEVICE_TABLE(usb, cdc_devs);

static struct usb_driver cdc_ncm_driver = {
        .name = "cdc_ncm",
        .id_table = cdc_devs,
        .probe = usbnet_probe,
        .disconnect = usbnet_disconnect,
        .suspend = usbnet_suspend,
        .resume = usbnet_resume,
        .reset_resume =        usbnet_resume,
        .supports_autosuspend = 1,
        .disable_hub_initiated_lpm = 1,
};

module_usb_driver(cdc_ncm_driver);

MODULE_AUTHOR("Hans Petter Selasky");
MODULE_DESCRIPTION("USB CDC NCM host driver");
MODULE_LICENSE("Dual BSD/GPL");































   69 






   71 
   88 

   12 
   17 






   12 



   58 



   12 
   59 






   71 








   47 

   24 


















   69 










   31 
   38 















   76 


   76 



   59 

















   55 
    4 
















   14 






   69 

    5 





   88 




   88 



   59 
    9 

   68 







   16 
    1 














   87 

   87 

   78 


   42 




   59 


   73 

   53 


   50 





   76 
   44 













   36 
   52 











   93 






   62 
   38 









   93 






   88 


    5 










   88 





   43 
   25 












   60 





    1 

   59 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
// SPDX-License-Identifier: GPL-2.0-only
/*
 * DVB USB library - provides a generic interface for a DVB USB device driver.
 *
 * dvb-usb-init.c
 *
 * Copyright (C) 2004-6 Patrick Boettcher (patrick.boettcher@posteo.de)
 *
 * see Documentation/driver-api/media/drivers/dvb-usb.rst for more information
 */
#include "dvb-usb-common.h"

/* debug */
int dvb_usb_debug;
module_param_named(debug, dvb_usb_debug, int, 0644);
MODULE_PARM_DESC(debug, "set debugging level (1=info,xfer=2,pll=4,ts=8,err=16,rc=32,fw=64,mem=128,uxfer=256  (or-able))." DVB_USB_DEBUG_STATUS);

int dvb_usb_disable_rc_polling;
module_param_named(disable_rc_polling, dvb_usb_disable_rc_polling, int, 0644);
MODULE_PARM_DESC(disable_rc_polling, "disable remote control polling (default: 0).");

static int dvb_usb_force_pid_filter_usage;
module_param_named(force_pid_filter_usage, dvb_usb_force_pid_filter_usage, int, 0444);
MODULE_PARM_DESC(force_pid_filter_usage, "force all dvb-usb-devices to use a PID filter, if any (default: 0).");

static int dvb_usb_adapter_init(struct dvb_usb_device *d, short *adapter_nrs)
{
        struct dvb_usb_adapter *adap;
        int ret, n, o;

        for (n = 0; n < d->props.num_adapters; n++) {
                adap = &d->adapter[n];
                adap->dev = d;
                adap->id  = n;

                memcpy(&adap->props, &d->props.adapter[n], sizeof(struct dvb_usb_adapter_properties));

                for (o = 0; o < adap->props.num_frontends; o++) {
                        struct dvb_usb_adapter_fe_properties *props = &adap->props.fe[o];
                        /* speed - when running at FULL speed we need a HW PID filter */
                        if (d->udev->speed == USB_SPEED_FULL && !(props->caps & DVB_USB_ADAP_HAS_PID_FILTER)) {
                                err("This USB2.0 device cannot be run on a USB1.1 port. (it lacks a hardware PID filter)");
                                return -ENODEV;
                        }

                        if ((d->udev->speed == USB_SPEED_FULL && props->caps & DVB_USB_ADAP_HAS_PID_FILTER) ||
                                (props->caps & DVB_USB_ADAP_NEED_PID_FILTERING)) {
                                info("will use the device's hardware PID filter (table count: %d).", props->pid_filter_count);
                                adap->fe_adap[o].pid_filtering  = 1;
                                adap->fe_adap[o].max_feed_count = props->pid_filter_count;
                        } else {
                                info("will pass the complete MPEG2 transport stream to the software demuxer.");
                                adap->fe_adap[o].pid_filtering  = 0;
                                adap->fe_adap[o].max_feed_count = 255;
                        }

                        if (!adap->fe_adap[o].pid_filtering &&
                                dvb_usb_force_pid_filter_usage &&
                                props->caps & DVB_USB_ADAP_HAS_PID_FILTER) {
                                info("pid filter enabled by module option.");
                                adap->fe_adap[o].pid_filtering  = 1;
                                adap->fe_adap[o].max_feed_count = props->pid_filter_count;
                        }

                        if (props->size_of_priv > 0) {
                                adap->fe_adap[o].priv = kzalloc(props->size_of_priv, GFP_KERNEL);
                                if (adap->fe_adap[o].priv == NULL) {
                                        err("no memory for priv for adapter %d fe %d.", n, o);
                                        return -ENOMEM;
                                }
                        }
                }

                if (adap->props.size_of_priv > 0) {
                        adap->priv = kzalloc(adap->props.size_of_priv, GFP_KERNEL);
                        if (adap->priv == NULL) {
                                err("no memory for priv for adapter %d.", n);
                                return -ENOMEM;
                        }
                }

                ret = dvb_usb_adapter_stream_init(adap);
                if (ret)
                        goto stream_init_err;

                ret = dvb_usb_adapter_dvb_init(adap, adapter_nrs);
                if (ret)
                        goto dvb_init_err;

                ret = dvb_usb_adapter_frontend_init(adap);
                if (ret)
                        goto frontend_init_err;

                /* use exclusive FE lock if there is multiple shared FEs */
                if (adap->fe_adap[1].fe && adap->dvb_adap.mfe_shared < 1)
                        adap->dvb_adap.mfe_shared = 1;

                d->num_adapters_initialized++;
                d->state |= DVB_USB_STATE_DVB;
        }

        /*
         * when reloading the driver w/o replugging the device
         * sometimes a timeout occurs, this helps
         */
        if (d->props.generic_bulk_ctrl_endpoint != 0) {
                usb_clear_halt(d->udev, usb_sndbulkpipe(d->udev, d->props.generic_bulk_ctrl_endpoint));
                usb_clear_halt(d->udev, usb_rcvbulkpipe(d->udev, d->props.generic_bulk_ctrl_endpoint));
        }

        return 0;

frontend_init_err:
        dvb_usb_adapter_dvb_exit(adap);
dvb_init_err:
        dvb_usb_adapter_stream_exit(adap);
stream_init_err:
        kfree(adap->priv);
        return ret;
}

static int dvb_usb_adapter_exit(struct dvb_usb_device *d)
{
        int n;

        for (n = 0; n < d->num_adapters_initialized; n++) {
                dvb_usb_adapter_frontend_exit(&d->adapter[n]);
                dvb_usb_adapter_dvb_exit(&d->adapter[n]);
                dvb_usb_adapter_stream_exit(&d->adapter[n]);
                kfree(d->adapter[n].priv);
        }
        d->num_adapters_initialized = 0;
        d->state &= ~DVB_USB_STATE_DVB;
        return 0;
}


/* general initialization functions */
static int dvb_usb_exit(struct dvb_usb_device *d)
{
        deb_info("state before exiting everything: %x\n", d->state);
        dvb_usb_remote_exit(d);
        dvb_usb_adapter_exit(d);
        dvb_usb_i2c_exit(d);
        deb_info("state should be zero now: %x\n", d->state);
        d->state = DVB_USB_STATE_INIT;

        if (d->priv != NULL && d->props.priv_destroy != NULL)
                d->props.priv_destroy(d);

        kfree(d->priv);
        kfree(d);
        return 0;
}

static int dvb_usb_init(struct dvb_usb_device *d, short *adapter_nums)
{
        int ret = 0;

        mutex_init(&d->data_mutex);
        mutex_init(&d->usb_mutex);
        mutex_init(&d->i2c_mutex);

        d->state = DVB_USB_STATE_INIT;

        if (d->props.size_of_priv > 0) {
                d->priv = kzalloc(d->props.size_of_priv, GFP_KERNEL);
                if (d->priv == NULL) {
                        err("no memory for priv in 'struct dvb_usb_device'");
                        return -ENOMEM;
                }

                if (d->props.priv_init != NULL) {
                        ret = d->props.priv_init(d);
                        if (ret != 0)
                                goto err_priv_init;
                }
        }

        /* check the capabilities and set appropriate variables */
        dvb_usb_device_power_ctrl(d, 1);

        ret = dvb_usb_i2c_init(d);
        if (ret)
                goto err_i2c_init;
        ret = dvb_usb_adapter_init(d, adapter_nums);
        if (ret)
                goto err_adapter_init;

        if ((ret = dvb_usb_remote_init(d)))
                err("could not initialize remote control.");

        dvb_usb_device_power_ctrl(d, 0);

        return 0;

err_adapter_init:
        dvb_usb_adapter_exit(d);
        dvb_usb_i2c_exit(d);
err_i2c_init:
        if (d->priv && d->props.priv_destroy)
                d->props.priv_destroy(d);
err_priv_init:
        kfree(d->priv);
        d->priv = NULL;
        return ret;
}

/* determine the name and the state of the just found USB device */
static const struct dvb_usb_device_description *dvb_usb_find_device(struct usb_device *udev, const struct dvb_usb_device_properties *props, int *cold)
{
        int i, j;
        const struct dvb_usb_device_description *desc = NULL;

        *cold = -1;

        for (i = 0; i < props->num_device_descs; i++) {

                for (j = 0; j < DVB_USB_ID_MAX_NUM && props->devices[i].cold_ids[j] != NULL; j++) {
                        deb_info("check for cold %x %x\n", props->devices[i].cold_ids[j]->idVendor, props->devices[i].cold_ids[j]->idProduct);
                        if (props->devices[i].cold_ids[j]->idVendor  == le16_to_cpu(udev->descriptor.idVendor) &&
                                props->devices[i].cold_ids[j]->idProduct == le16_to_cpu(udev->descriptor.idProduct)) {
                                *cold = 1;
                                desc = &props->devices[i];
                                break;
                        }
                }

                if (desc != NULL)
                        break;

                for (j = 0; j < DVB_USB_ID_MAX_NUM && props->devices[i].warm_ids[j] != NULL; j++) {
                        deb_info("check for warm %x %x\n", props->devices[i].warm_ids[j]->idVendor, props->devices[i].warm_ids[j]->idProduct);
                        if (props->devices[i].warm_ids[j]->idVendor == le16_to_cpu(udev->descriptor.idVendor) &&
                                props->devices[i].warm_ids[j]->idProduct == le16_to_cpu(udev->descriptor.idProduct)) {
                                *cold = 0;
                                desc = &props->devices[i];
                                break;
                        }
                }
        }

        if (desc != NULL && props->identify_state != NULL)
                props->identify_state(udev, props, &desc, cold);

        return desc;
}

int dvb_usb_device_power_ctrl(struct dvb_usb_device *d, int onoff)
{
        if (onoff)
                d->powered++;
        else
                d->powered--;

        if (d->powered == 0 || (onoff && d->powered == 1)) { /* when switching from 1 to 0 or from 0 to 1 */
                deb_info("power control: %d\n", onoff);
                if (d->props.power_ctrl)
                        return d->props.power_ctrl(d, onoff);
        }
        return 0;
}

/*
 * USB
 */
int dvb_usb_device_init(struct usb_interface *intf,
                        const struct dvb_usb_device_properties *props,
                        struct module *owner, struct dvb_usb_device **du,
                        short *adapter_nums)
{
        struct usb_device *udev = interface_to_usbdev(intf);
        struct dvb_usb_device *d = NULL;
        const struct dvb_usb_device_description *desc = NULL;

        int ret = -ENOMEM, cold = 0;

        if (du != NULL)
                *du = NULL;

        d = kzalloc(sizeof(*d), GFP_KERNEL);
        if (!d) {
                err("no memory for 'struct dvb_usb_device'");
                return -ENOMEM;
        }

        memcpy(&d->props, props, sizeof(struct dvb_usb_device_properties));

        desc = dvb_usb_find_device(udev, &d->props, &cold);
        if (!desc) {
                deb_err("something went very wrong, device was not found in current device list - let's see what comes next.\n");
                ret = -ENODEV;
                goto error;
        }

        if (cold) {
                info("found a '%s' in cold state, will try to load a firmware", desc->name);
                ret = dvb_usb_download_firmware(udev, props);
                if (!props->no_reconnect || ret != 0)
                        goto error;
        }

        info("found a '%s' in warm state.", desc->name);
        d->udev = udev;
        d->desc = desc;
        d->owner = owner;

        usb_set_intfdata(intf, d);

        ret = dvb_usb_init(d, adapter_nums);
        if (ret) {
                info("%s error while loading driver (%d)", desc->name, ret);
                goto error;
        }

        if (du)
                *du = d;

        info("%s successfully initialized and connected.", desc->name);
        return 0;

 error:
        usb_set_intfdata(intf, NULL);
        kfree(d);
        return ret;
}
EXPORT_SYMBOL(dvb_usb_device_init);

void dvb_usb_device_exit(struct usb_interface *intf)
{
        struct dvb_usb_device *d = usb_get_intfdata(intf);
        const char *default_name = "generic DVB-USB module";
        char name[40];

        usb_set_intfdata(intf, NULL);
        if (d != NULL && d->desc != NULL) {
                strscpy(name, d->desc->name, sizeof(name));
                dvb_usb_exit(d);
        } else {
                strscpy(name, default_name, sizeof(name));
        }
        info("%s successfully deinitialized and disconnected.", name);

}
EXPORT_SYMBOL(dvb_usb_device_exit);

MODULE_VERSION("1.0");
MODULE_AUTHOR("Patrick Boettcher <patrick.boettcher@posteo.de>");
MODULE_DESCRIPTION("A library module containing commonly used USB and DVB function USB DVB devices");
MODULE_LICENSE("GPL");






































    1 
































    1 

















































































  218 





    5 




  213 









  213 



































   82 
   36 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
/*
 *  include/linux/ktime.h
 *
 *  ktime_t - nanosecond-resolution time format.
 *
 *   Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de>
 *   Copyright(C) 2005, Red Hat, Inc., Ingo Molnar
 *
 *  data type definitions, declarations, prototypes and macros.
 *
 *  Started by: Thomas Gleixner and Ingo Molnar
 *
 *  Credits:
 *
 *          Roman Zippel provided the ideas and primary code snippets of
 *          the ktime_t union and further simplifications of the original
 *          code.
 *
 *  For licencing details see kernel-base/COPYING
 */
#ifndef _LINUX_KTIME_H
#define _LINUX_KTIME_H

#include <asm/bug.h>
#include <linux/jiffies.h>
#include <linux/time.h>
#include <linux/types.h>

/**
 * ktime_set - Set a ktime_t variable from a seconds/nanoseconds value
 * @secs:        seconds to set
 * @nsecs:        nanoseconds to set
 *
 * Return: The ktime_t representation of the value.
 */
static inline ktime_t ktime_set(const s64 secs, const unsigned long nsecs)
{
        if (unlikely(secs >= KTIME_SEC_MAX))
                return KTIME_MAX;

        return secs * NSEC_PER_SEC + (s64)nsecs;
}

/* Subtract two ktime_t variables. rem = lhs -rhs: */
#define ktime_sub(lhs, rhs)        ((lhs) - (rhs))

/* Add two ktime_t variables. res = lhs + rhs: */
#define ktime_add(lhs, rhs)        ((lhs) + (rhs))

/*
 * Same as ktime_add(), but avoids undefined behaviour on overflow; however,
 * this means that you must check the result for overflow yourself.
 */
#define ktime_add_unsafe(lhs, rhs)        ((u64) (lhs) + (rhs))

/*
 * Add a ktime_t variable and a scalar nanosecond value.
 * res = kt + nsval:
 */
#define ktime_add_ns(kt, nsval)                ((kt) + (nsval))

/*
 * Subtract a scalar nanosecod from a ktime_t variable
 * res = kt - nsval:
 */
#define ktime_sub_ns(kt, nsval)                ((kt) - (nsval))

/* convert a timespec64 to ktime_t format: */
static inline ktime_t timespec64_to_ktime(struct timespec64 ts)
{
        return ktime_set(ts.tv_sec, ts.tv_nsec);
}

/* Map the ktime_t to timespec conversion to ns_to_timespec function */
#define ktime_to_timespec64(kt)                ns_to_timespec64((kt))

/* Convert ktime_t to nanoseconds */
static inline s64 ktime_to_ns(const ktime_t kt)
{
        return kt;
}

/**
 * ktime_compare - Compares two ktime_t variables for less, greater or equal
 * @cmp1:        comparable1
 * @cmp2:        comparable2
 *
 * Return: ...
 *   cmp1  < cmp2: return <0
 *   cmp1 == cmp2: return 0
 *   cmp1  > cmp2: return >0
 */
static inline int ktime_compare(const ktime_t cmp1, const ktime_t cmp2)
{
        if (cmp1 < cmp2)
                return -1;
        if (cmp1 > cmp2)
                return 1;
        return 0;
}

/**
 * ktime_after - Compare if a ktime_t value is bigger than another one.
 * @cmp1:        comparable1
 * @cmp2:        comparable2
 *
 * Return: true if cmp1 happened after cmp2.
 */
static inline bool ktime_after(const ktime_t cmp1, const ktime_t cmp2)
{
        return ktime_compare(cmp1, cmp2) > 0;
}

/**
 * ktime_before - Compare if a ktime_t value is smaller than another one.
 * @cmp1:        comparable1
 * @cmp2:        comparable2
 *
 * Return: true if cmp1 happened before cmp2.
 */
static inline bool ktime_before(const ktime_t cmp1, const ktime_t cmp2)
{
        return ktime_compare(cmp1, cmp2) < 0;
}

#if BITS_PER_LONG < 64
extern s64 __ktime_divns(const ktime_t kt, s64 div);
static inline s64 ktime_divns(const ktime_t kt, s64 div)
{
        /*
         * Negative divisors could cause an inf loop,
         * so bug out here.
         */
        BUG_ON(div < 0);
        if (__builtin_constant_p(div) && !(div >> 32)) {
                s64 ns = kt;
                u64 tmp = ns < 0 ? -ns : ns;

                do_div(tmp, div);
                return ns < 0 ? -tmp : tmp;
        } else {
                return __ktime_divns(kt, div);
        }
}
#else /* BITS_PER_LONG < 64 */
static inline s64 ktime_divns(const ktime_t kt, s64 div)
{
        /*
         * 32-bit implementation cannot handle negative divisors,
         * so catch them on 64bit as well.
         */
        WARN_ON(div < 0);
        return kt / div;
}
#endif

static inline s64 ktime_to_us(const ktime_t kt)
{
        return ktime_divns(kt, NSEC_PER_USEC);
}

static inline s64 ktime_to_ms(const ktime_t kt)
{
        return ktime_divns(kt, NSEC_PER_MSEC);
}

static inline s64 ktime_us_delta(const ktime_t later, const ktime_t earlier)
{
       return ktime_to_us(ktime_sub(later, earlier));
}

static inline s64 ktime_ms_delta(const ktime_t later, const ktime_t earlier)
{
        return ktime_to_ms(ktime_sub(later, earlier));
}

static inline ktime_t ktime_add_us(const ktime_t kt, const u64 usec)
{
        return ktime_add_ns(kt, usec * NSEC_PER_USEC);
}

static inline ktime_t ktime_add_ms(const ktime_t kt, const u64 msec)
{
        return ktime_add_ns(kt, msec * NSEC_PER_MSEC);
}

static inline ktime_t ktime_sub_us(const ktime_t kt, const u64 usec)
{
        return ktime_sub_ns(kt, usec * NSEC_PER_USEC);
}

static inline ktime_t ktime_sub_ms(const ktime_t kt, const u64 msec)
{
        return ktime_sub_ns(kt, msec * NSEC_PER_MSEC);
}

extern ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs);

/**
 * ktime_to_timespec64_cond - convert a ktime_t variable to timespec64
 *                            format only if the variable contains data
 * @kt:                the ktime_t variable to convert
 * @ts:                the timespec variable to store the result in
 *
 * Return: %true if there was a successful conversion, %false if kt was 0.
 */
static inline __must_check bool ktime_to_timespec64_cond(const ktime_t kt,
                                                       struct timespec64 *ts)
{
        if (kt) {
                *ts = ktime_to_timespec64(kt);
                return true;
        } else {
                return false;
        }
}

#include <vdso/ktime.h>

static inline ktime_t ns_to_ktime(u64 ns)
{
        return ns;
}

static inline ktime_t us_to_ktime(u64 us)
{
        return us * NSEC_PER_USEC;
}

static inline ktime_t ms_to_ktime(u64 ms)
{
        return ms * NSEC_PER_MSEC;
}

# include <linux/timekeeping.h>

#endif





























    2 





    5 
    5 
   14 












   16 
   16 

















    6 





















    6 






















    6 





















    6 






















    6 



    6 



    6 



    6 



    6 


















    6 
















  163 
    1 

  163 





  129 



  130 





    6 




    6 







    6 
































    6 























    6 







































    6 















































































































    6 








    6 


    6 


    6 


    6 








    6 
    6 









    6 















    6 



    1 
    6 






    6 












    6 
















    8 







    8 












    2 

    2 








    3 




    3 
    3 



    3 
    3 



    2 
















    2 





    2 





    2 















    3 





































































































































































































































































































































































    3 







    8 


















    3 









































    3 




    3 

    3 












    3 

























    3 
















    3 
    3 
    3 

    3 


    3 












    3 

















    5 












    5 

















    8 


    8 


    3 

    8 

    5 







    6 
    5 



    5 





























































 2623 




 2579 



   99 





    3 




    3 






    5 









    6 






    3 



















    3 











    3 
    3 
    3 


    3 
















































































































































  163 



  162 

  163 























































 2601 






 2601 


 2550 


  163 




 2609 







 2555 


  161 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
 */

#include "devl_internal.h"

#define DEVLINK_PORT_FN_CAPS_VALID_MASK \
        (_BITUL(__DEVLINK_PORT_FN_ATTR_CAPS_MAX) - 1)

static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = {
        [DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY },
        [DEVLINK_PORT_FN_ATTR_STATE] =
                NLA_POLICY_RANGE(NLA_U8, DEVLINK_PORT_FN_STATE_INACTIVE,
                                 DEVLINK_PORT_FN_STATE_ACTIVE),
        [DEVLINK_PORT_FN_ATTR_CAPS] =
                NLA_POLICY_BITFIELD32(DEVLINK_PORT_FN_CAPS_VALID_MASK),
        [DEVLINK_PORT_FN_ATTR_MAX_IO_EQS] = { .type = NLA_U32 },
};

#define ASSERT_DEVLINK_PORT_REGISTERED(devlink_port)                                \
        WARN_ON_ONCE(!(devlink_port)->registered)
#define ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port)                        \
        WARN_ON_ONCE((devlink_port)->registered)

struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
                                               unsigned int port_index)
{
        return xa_load(&devlink->ports, port_index);
}

struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
                                                 struct nlattr **attrs)
{
        if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
                u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
                struct devlink_port *devlink_port;

                devlink_port = devlink_port_get_by_index(devlink, port_index);
                if (!devlink_port)
                        return ERR_PTR(-ENODEV);
                return devlink_port;
        }
        return ERR_PTR(-EINVAL);
}

struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
                                                struct genl_info *info)
{
        return devlink_port_get_from_attrs(devlink, info->attrs);
}

static void devlink_port_fn_cap_fill(struct nla_bitfield32 *caps,
                                     u32 cap, bool is_enable)
{
        caps->selector |= cap;
        if (is_enable)
                caps->value |= cap;
}

static int devlink_port_fn_roce_fill(struct devlink_port *devlink_port,
                                     struct nla_bitfield32 *caps,
                                     struct netlink_ext_ack *extack)
{
        bool is_enable;
        int err;

        if (!devlink_port->ops->port_fn_roce_get)
                return 0;

        err = devlink_port->ops->port_fn_roce_get(devlink_port, &is_enable,
                                                  extack);
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }

        devlink_port_fn_cap_fill(caps, DEVLINK_PORT_FN_CAP_ROCE, is_enable);
        return 0;
}

static int devlink_port_fn_migratable_fill(struct devlink_port *devlink_port,
                                           struct nla_bitfield32 *caps,
                                           struct netlink_ext_ack *extack)
{
        bool is_enable;
        int err;

        if (!devlink_port->ops->port_fn_migratable_get ||
            devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_VF)
                return 0;

        err = devlink_port->ops->port_fn_migratable_get(devlink_port,
                                                        &is_enable, extack);
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }

        devlink_port_fn_cap_fill(caps, DEVLINK_PORT_FN_CAP_MIGRATABLE, is_enable);
        return 0;
}

static int devlink_port_fn_ipsec_crypto_fill(struct devlink_port *devlink_port,
                                             struct nla_bitfield32 *caps,
                                             struct netlink_ext_ack *extack)
{
        bool is_enable;
        int err;

        if (!devlink_port->ops->port_fn_ipsec_crypto_get ||
            devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_VF)
                return 0;

        err = devlink_port->ops->port_fn_ipsec_crypto_get(devlink_port, &is_enable, extack);
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }

        devlink_port_fn_cap_fill(caps, DEVLINK_PORT_FN_CAP_IPSEC_CRYPTO, is_enable);
        return 0;
}

static int devlink_port_fn_ipsec_packet_fill(struct devlink_port *devlink_port,
                                             struct nla_bitfield32 *caps,
                                             struct netlink_ext_ack *extack)
{
        bool is_enable;
        int err;

        if (!devlink_port->ops->port_fn_ipsec_packet_get ||
            devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_VF)
                return 0;

        err = devlink_port->ops->port_fn_ipsec_packet_get(devlink_port, &is_enable, extack);
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }

        devlink_port_fn_cap_fill(caps, DEVLINK_PORT_FN_CAP_IPSEC_PACKET, is_enable);
        return 0;
}

static int devlink_port_fn_caps_fill(struct devlink_port *devlink_port,
                                     struct sk_buff *msg,
                                     struct netlink_ext_ack *extack,
                                     bool *msg_updated)
{
        struct nla_bitfield32 caps = {};
        int err;

        err = devlink_port_fn_roce_fill(devlink_port, &caps, extack);
        if (err)
                return err;

        err = devlink_port_fn_migratable_fill(devlink_port, &caps, extack);
        if (err)
                return err;

        err = devlink_port_fn_ipsec_crypto_fill(devlink_port, &caps, extack);
        if (err)
                return err;

        err = devlink_port_fn_ipsec_packet_fill(devlink_port, &caps, extack);
        if (err)
                return err;

        if (!caps.selector)
                return 0;
        err = nla_put_bitfield32(msg, DEVLINK_PORT_FN_ATTR_CAPS, caps.value,
                                 caps.selector);
        if (err)
                return err;

        *msg_updated = true;
        return 0;
}

static int devlink_port_fn_max_io_eqs_fill(struct devlink_port *port,
                                           struct sk_buff *msg,
                                           struct netlink_ext_ack *extack,
                                           bool *msg_updated)
{
        u32 max_io_eqs;
        int err;

        if (!port->ops->port_fn_max_io_eqs_get)
                return 0;

        err = port->ops->port_fn_max_io_eqs_get(port, &max_io_eqs, extack);
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }
        err = nla_put_u32(msg, DEVLINK_PORT_FN_ATTR_MAX_IO_EQS, max_io_eqs);
        if (err)
                return err;
        *msg_updated = true;
        return 0;
}

int devlink_nl_port_handle_fill(struct sk_buff *msg, struct devlink_port *devlink_port)
{
        if (devlink_nl_put_handle(msg, devlink_port->devlink))
                return -EMSGSIZE;
        if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
                return -EMSGSIZE;
        return 0;
}

size_t devlink_nl_port_handle_size(struct devlink_port *devlink_port)
{
        struct devlink *devlink = devlink_port->devlink;

        return nla_total_size(strlen(devlink->dev->bus->name) + 1) /* DEVLINK_ATTR_BUS_NAME */
             + nla_total_size(strlen(dev_name(devlink->dev)) + 1) /* DEVLINK_ATTR_DEV_NAME */
             + nla_total_size(4); /* DEVLINK_ATTR_PORT_INDEX */
}

static int devlink_nl_port_attrs_put(struct sk_buff *msg,
                                     struct devlink_port *devlink_port)
{
        struct devlink_port_attrs *attrs = &devlink_port->attrs;

        if (!devlink_port->attrs_set)
                return 0;
        if (attrs->lanes) {
                if (nla_put_u32(msg, DEVLINK_ATTR_PORT_LANES, attrs->lanes))
                        return -EMSGSIZE;
        }
        if (nla_put_u8(msg, DEVLINK_ATTR_PORT_SPLITTABLE, attrs->splittable))
                return -EMSGSIZE;
        if (nla_put_u16(msg, DEVLINK_ATTR_PORT_FLAVOUR, attrs->flavour))
                return -EMSGSIZE;
        switch (devlink_port->attrs.flavour) {
        case DEVLINK_PORT_FLAVOUR_PCI_PF:
                if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
                                attrs->pci_pf.controller) ||
                    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_pf.pf))
                        return -EMSGSIZE;
                if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_pf.external))
                        return -EMSGSIZE;
                break;
        case DEVLINK_PORT_FLAVOUR_PCI_VF:
                if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
                                attrs->pci_vf.controller) ||
                    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_vf.pf) ||
                    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER, attrs->pci_vf.vf))
                        return -EMSGSIZE;
                if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_vf.external))
                        return -EMSGSIZE;
                break;
        case DEVLINK_PORT_FLAVOUR_PCI_SF:
                if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
                                attrs->pci_sf.controller) ||
                    nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
                                attrs->pci_sf.pf) ||
                    nla_put_u32(msg, DEVLINK_ATTR_PORT_PCI_SF_NUMBER,
                                attrs->pci_sf.sf))
                        return -EMSGSIZE;
                break;
        case DEVLINK_PORT_FLAVOUR_PHYSICAL:
        case DEVLINK_PORT_FLAVOUR_CPU:
        case DEVLINK_PORT_FLAVOUR_DSA:
                if (nla_put_u32(msg, DEVLINK_ATTR_PORT_NUMBER,
                                attrs->phys.port_number))
                        return -EMSGSIZE;
                if (!attrs->split)
                        return 0;
                if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
                                attrs->phys.port_number))
                        return -EMSGSIZE;
                if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER,
                                attrs->phys.split_subport_number))
                        return -EMSGSIZE;
                break;
        default:
                break;
        }
        return 0;
}

static int devlink_port_fn_hw_addr_fill(struct devlink_port *port,
                                        struct sk_buff *msg,
                                        struct netlink_ext_ack *extack,
                                        bool *msg_updated)
{
        u8 hw_addr[MAX_ADDR_LEN];
        int hw_addr_len;
        int err;

        if (!port->ops->port_fn_hw_addr_get)
                return 0;

        err = port->ops->port_fn_hw_addr_get(port, hw_addr, &hw_addr_len,
                                             extack);
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }
        err = nla_put(msg, DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR, hw_addr_len, hw_addr);
        if (err)
                return err;
        *msg_updated = true;
        return 0;
}

static bool
devlink_port_fn_state_valid(enum devlink_port_fn_state state)
{
        return state == DEVLINK_PORT_FN_STATE_INACTIVE ||
               state == DEVLINK_PORT_FN_STATE_ACTIVE;
}

static bool
devlink_port_fn_opstate_valid(enum devlink_port_fn_opstate opstate)
{
        return opstate == DEVLINK_PORT_FN_OPSTATE_DETACHED ||
               opstate == DEVLINK_PORT_FN_OPSTATE_ATTACHED;
}

static int devlink_port_fn_state_fill(struct devlink_port *port,
                                      struct sk_buff *msg,
                                      struct netlink_ext_ack *extack,
                                      bool *msg_updated)
{
        enum devlink_port_fn_opstate opstate;
        enum devlink_port_fn_state state;
        int err;

        if (!port->ops->port_fn_state_get)
                return 0;

        err = port->ops->port_fn_state_get(port, &state, &opstate, extack);
        if (err) {
                if (err == -EOPNOTSUPP)
                        return 0;
                return err;
        }
        if (!devlink_port_fn_state_valid(state)) {
                WARN_ON_ONCE(1);
                NL_SET_ERR_MSG(extack, "Invalid state read from driver");
                return -EINVAL;
        }
        if (!devlink_port_fn_opstate_valid(opstate)) {
                WARN_ON_ONCE(1);
                NL_SET_ERR_MSG(extack, "Invalid operational state read from driver");
                return -EINVAL;
        }
        if (nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_STATE, state) ||
            nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_OPSTATE, opstate))
                return -EMSGSIZE;
        *msg_updated = true;
        return 0;
}

static int
devlink_port_fn_mig_set(struct devlink_port *devlink_port, bool enable,
                        struct netlink_ext_ack *extack)
{
        return devlink_port->ops->port_fn_migratable_set(devlink_port, enable,
                                                         extack);
}

static int
devlink_port_fn_roce_set(struct devlink_port *devlink_port, bool enable,
                         struct netlink_ext_ack *extack)
{
        return devlink_port->ops->port_fn_roce_set(devlink_port, enable,
                                                   extack);
}

static int
devlink_port_fn_ipsec_crypto_set(struct devlink_port *devlink_port, bool enable,
                                 struct netlink_ext_ack *extack)
{
        return devlink_port->ops->port_fn_ipsec_crypto_set(devlink_port, enable, extack);
}

static int
devlink_port_fn_ipsec_packet_set(struct devlink_port *devlink_port, bool enable,
                                 struct netlink_ext_ack *extack)
{
        return devlink_port->ops->port_fn_ipsec_packet_set(devlink_port, enable, extack);
}

static int devlink_port_fn_caps_set(struct devlink_port *devlink_port,
                                    const struct nlattr *attr,
                                    struct netlink_ext_ack *extack)
{
        struct nla_bitfield32 caps;
        u32 caps_value;
        int err;

        caps = nla_get_bitfield32(attr);
        caps_value = caps.value & caps.selector;
        if (caps.selector & DEVLINK_PORT_FN_CAP_ROCE) {
                err = devlink_port_fn_roce_set(devlink_port,
                                               caps_value & DEVLINK_PORT_FN_CAP_ROCE,
                                               extack);
                if (err)
                        return err;
        }
        if (caps.selector & DEVLINK_PORT_FN_CAP_MIGRATABLE) {
                err = devlink_port_fn_mig_set(devlink_port, caps_value &
                                              DEVLINK_PORT_FN_CAP_MIGRATABLE,
                                              extack);
                if (err)
                        return err;
        }
        if (caps.selector & DEVLINK_PORT_FN_CAP_IPSEC_CRYPTO) {
                err = devlink_port_fn_ipsec_crypto_set(devlink_port, caps_value &
                                                       DEVLINK_PORT_FN_CAP_IPSEC_CRYPTO,
                                                       extack);
                if (err)
                        return err;
        }
        if (caps.selector & DEVLINK_PORT_FN_CAP_IPSEC_PACKET) {
                err = devlink_port_fn_ipsec_packet_set(devlink_port, caps_value &
                                                       DEVLINK_PORT_FN_CAP_IPSEC_PACKET,
                                                       extack);
                if (err)
                        return err;
        }
        return 0;
}

static int
devlink_port_fn_max_io_eqs_set(struct devlink_port *devlink_port,
                               const struct nlattr *attr,
                               struct netlink_ext_ack *extack)
{
        u32 max_io_eqs;

        max_io_eqs = nla_get_u32(attr);
        return devlink_port->ops->port_fn_max_io_eqs_set(devlink_port,
                                                         max_io_eqs, extack);
}

static int
devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *port,
                                   struct netlink_ext_ack *extack)
{
        struct nlattr *function_attr;
        bool msg_updated = false;
        int err;

        function_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PORT_FUNCTION);
        if (!function_attr)
                return -EMSGSIZE;

        err = devlink_port_fn_hw_addr_fill(port, msg, extack, &msg_updated);
        if (err)
                goto out;
        err = devlink_port_fn_caps_fill(port, msg, extack, &msg_updated);
        if (err)
                goto out;
        err = devlink_port_fn_state_fill(port, msg, extack, &msg_updated);
        if (err)
                goto out;
        err = devlink_port_fn_max_io_eqs_fill(port, msg, extack, &msg_updated);
        if (err)
                goto out;
        err = devlink_rel_devlink_handle_put(msg, port->devlink,
                                             port->rel_index,
                                             DEVLINK_PORT_FN_ATTR_DEVLINK,
                                             &msg_updated);

out:
        if (err || !msg_updated)
                nla_nest_cancel(msg, function_attr);
        else
                nla_nest_end(msg, function_attr);
        return err;
}

static int devlink_nl_port_fill(struct sk_buff *msg,
                                struct devlink_port *devlink_port,
                                enum devlink_command cmd, u32 portid, u32 seq,
                                int flags, struct netlink_ext_ack *extack)
{
        struct devlink *devlink = devlink_port->devlink;
        void *hdr;

        hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
        if (!hdr)
                return -EMSGSIZE;

        if (devlink_nl_put_handle(msg, devlink))
                goto nla_put_failure;
        if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
                goto nla_put_failure;

        spin_lock_bh(&devlink_port->type_lock);
        if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
                goto nla_put_failure_type_locked;
        if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
            nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
                        devlink_port->desired_type))
                goto nla_put_failure_type_locked;
        if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
                if (devlink_port->type_eth.netdev &&
                    (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
                                 devlink_port->type_eth.ifindex) ||
                     nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
                                    devlink_port->type_eth.ifname)))
                        goto nla_put_failure_type_locked;
        }
        if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
                struct ib_device *ibdev = devlink_port->type_ib.ibdev;

                if (ibdev &&
                    nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
                                   ibdev->name))
                        goto nla_put_failure_type_locked;
        }
        spin_unlock_bh(&devlink_port->type_lock);
        if (devlink_nl_port_attrs_put(msg, devlink_port))
                goto nla_put_failure;
        if (devlink_nl_port_function_attrs_put(msg, devlink_port, extack))
                goto nla_put_failure;
        if (devlink_port->linecard &&
            nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX,
                        devlink_linecard_index(devlink_port->linecard)))
                goto nla_put_failure;

        genlmsg_end(msg, hdr);
        return 0;

nla_put_failure_type_locked:
        spin_unlock_bh(&devlink_port->type_lock);
nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

static void devlink_port_notify(struct devlink_port *devlink_port,
                                enum devlink_command cmd)
{
        struct devlink *devlink = devlink_port->devlink;
        struct devlink_obj_desc desc;
        struct sk_buff *msg;
        int err;

        WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);

        if (!__devl_is_registered(devlink) || !devlink_nl_notify_need(devlink))
                return;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return;

        err = devlink_nl_port_fill(msg, devlink_port, cmd, 0, 0, 0, NULL);
        if (err) {
                nlmsg_free(msg);
                return;
        }

        devlink_nl_obj_desc_init(&desc, devlink);
        devlink_nl_obj_desc_port_set(&desc, devlink_port);
        devlink_nl_notify_send_desc(devlink, msg, &desc);
}

static void devlink_ports_notify(struct devlink *devlink,
                                 enum devlink_command cmd)
{
        struct devlink_port *devlink_port;
        unsigned long port_index;

        xa_for_each(&devlink->ports, port_index, devlink_port)
                devlink_port_notify(devlink_port, cmd);
}

void devlink_ports_notify_register(struct devlink *devlink)
{
        devlink_ports_notify(devlink, DEVLINK_CMD_PORT_NEW);
}

void devlink_ports_notify_unregister(struct devlink *devlink)
{
        devlink_ports_notify(devlink, DEVLINK_CMD_PORT_DEL);
}

int devlink_nl_port_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink_port *devlink_port = info->user_ptr[1];
        struct sk_buff *msg;
        int err;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_PORT_NEW,
                                   info->snd_portid, info->snd_seq, 0,
                                   info->extack);
        if (err) {
                nlmsg_free(msg);
                return err;
        }

        return genlmsg_reply(msg, info);
}

static int
devlink_nl_port_get_dump_one(struct sk_buff *msg, struct devlink *devlink,
                             struct netlink_callback *cb, int flags)
{
        struct devlink_nl_dump_state *state = devlink_dump_state(cb);
        struct devlink_port *devlink_port;
        unsigned long port_index;
        int err = 0;

        xa_for_each_start(&devlink->ports, port_index, devlink_port, state->idx) {
                err = devlink_nl_port_fill(msg, devlink_port,
                                           DEVLINK_CMD_PORT_NEW,
                                           NETLINK_CB(cb->skb).portid,
                                           cb->nlh->nlmsg_seq, flags,
                                           cb->extack);
                if (err) {
                        state->idx = port_index;
                        break;
                }
        }

        return err;
}

int devlink_nl_port_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
{
        return devlink_nl_dumpit(skb, cb, devlink_nl_port_get_dump_one);
}

static int devlink_port_type_set(struct devlink_port *devlink_port,
                                 enum devlink_port_type port_type)

{
        int err;

        if (!devlink_port->ops->port_type_set)
                return -EOPNOTSUPP;

        if (port_type == devlink_port->type)
                return 0;

        err = devlink_port->ops->port_type_set(devlink_port, port_type);
        if (err)
                return err;

        devlink_port->desired_type = port_type;
        devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
        return 0;
}

static int devlink_port_function_hw_addr_set(struct devlink_port *port,
                                             const struct nlattr *attr,
                                             struct netlink_ext_ack *extack)
{
        const u8 *hw_addr;
        int hw_addr_len;

        hw_addr = nla_data(attr);
        hw_addr_len = nla_len(attr);
        if (hw_addr_len > MAX_ADDR_LEN) {
                NL_SET_ERR_MSG(extack, "Port function hardware address too long");
                return -EINVAL;
        }
        if (port->type == DEVLINK_PORT_TYPE_ETH) {
                if (hw_addr_len != ETH_ALEN) {
                        NL_SET_ERR_MSG(extack, "Address must be 6 bytes for Ethernet device");
                        return -EINVAL;
                }
                if (!is_unicast_ether_addr(hw_addr)) {
                        NL_SET_ERR_MSG(extack, "Non-unicast hardware address unsupported");
                        return -EINVAL;
                }
        }

        return port->ops->port_fn_hw_addr_set(port, hw_addr, hw_addr_len,
                                              extack);
}

static int devlink_port_fn_state_set(struct devlink_port *port,
                                     const struct nlattr *attr,
                                     struct netlink_ext_ack *extack)
{
        enum devlink_port_fn_state state;

        state = nla_get_u8(attr);
        return port->ops->port_fn_state_set(port, state, extack);
}

static int devlink_port_function_validate(struct devlink_port *devlink_port,
                                          struct nlattr **tb,
                                          struct netlink_ext_ack *extack)
{
        const struct devlink_port_ops *ops = devlink_port->ops;
        struct nlattr *attr;

        if (tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] &&
            !ops->port_fn_hw_addr_set) {
                NL_SET_ERR_MSG_ATTR(extack, tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR],
                                    "Port doesn't support function attributes");
                return -EOPNOTSUPP;
        }
        if (tb[DEVLINK_PORT_FN_ATTR_STATE] && !ops->port_fn_state_set) {
                NL_SET_ERR_MSG_ATTR(extack, tb[DEVLINK_PORT_FN_ATTR_STATE],
                                    "Function does not support state setting");
                return -EOPNOTSUPP;
        }
        attr = tb[DEVLINK_PORT_FN_ATTR_CAPS];
        if (attr) {
                struct nla_bitfield32 caps;

                caps = nla_get_bitfield32(attr);
                if (caps.selector & DEVLINK_PORT_FN_CAP_ROCE &&
                    !ops->port_fn_roce_set) {
                        NL_SET_ERR_MSG_ATTR(extack, attr,
                                            "Port doesn't support RoCE function attribute");
                        return -EOPNOTSUPP;
                }
                if (caps.selector & DEVLINK_PORT_FN_CAP_MIGRATABLE) {
                        if (!ops->port_fn_migratable_set) {
                                NL_SET_ERR_MSG_ATTR(extack, attr,
                                                    "Port doesn't support migratable function attribute");
                                return -EOPNOTSUPP;
                        }
                        if (devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_VF) {
                                NL_SET_ERR_MSG_ATTR(extack, attr,
                                                    "migratable function attribute supported for VFs only");
                                return -EOPNOTSUPP;
                        }
                }
                if (caps.selector & DEVLINK_PORT_FN_CAP_IPSEC_CRYPTO) {
                        if (!ops->port_fn_ipsec_crypto_set) {
                                NL_SET_ERR_MSG_ATTR(extack, attr,
                                                    "Port doesn't support ipsec_crypto function attribute");
                                return -EOPNOTSUPP;
                        }
                        if (devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_VF) {
                                NL_SET_ERR_MSG_ATTR(extack, attr,
                                                    "ipsec_crypto function attribute supported for VFs only");
                                return -EOPNOTSUPP;
                        }
                }
                if (caps.selector & DEVLINK_PORT_FN_CAP_IPSEC_PACKET) {
                        if (!ops->port_fn_ipsec_packet_set) {
                                NL_SET_ERR_MSG_ATTR(extack, attr,
                                                    "Port doesn't support ipsec_packet function attribute");
                                return -EOPNOTSUPP;
                        }
                        if (devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_VF) {
                                NL_SET_ERR_MSG_ATTR(extack, attr,
                                                    "ipsec_packet function attribute supported for VFs only");
                                return -EOPNOTSUPP;
                        }
                }
        }
        if (tb[DEVLINK_PORT_FN_ATTR_MAX_IO_EQS] &&
            !ops->port_fn_max_io_eqs_set) {
                NL_SET_ERR_MSG_ATTR(extack, tb[DEVLINK_PORT_FN_ATTR_MAX_IO_EQS],
                                    "Function does not support max_io_eqs setting");
                return -EOPNOTSUPP;
        }
        return 0;
}

static int devlink_port_function_set(struct devlink_port *port,
                                     const struct nlattr *attr,
                                     struct netlink_ext_ack *extack)
{
        struct nlattr *tb[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1];
        int err;

        err = nla_parse_nested(tb, DEVLINK_PORT_FUNCTION_ATTR_MAX, attr,
                               devlink_function_nl_policy, extack);
        if (err < 0) {
                NL_SET_ERR_MSG(extack, "Fail to parse port function attributes");
                return err;
        }

        err = devlink_port_function_validate(port, tb, extack);
        if (err)
                return err;

        attr = tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR];
        if (attr) {
                err = devlink_port_function_hw_addr_set(port, attr, extack);
                if (err)
                        return err;
        }

        attr = tb[DEVLINK_PORT_FN_ATTR_CAPS];
        if (attr) {
                err = devlink_port_fn_caps_set(port, attr, extack);
                if (err)
                        return err;
        }

        attr = tb[DEVLINK_PORT_FN_ATTR_MAX_IO_EQS];
        if (attr) {
                err = devlink_port_fn_max_io_eqs_set(port, attr, extack);
                if (err)
                        return err;
        }

        /* Keep this as the last function attribute set, so that when
         * multiple port function attributes are set along with state,
         * Those can be applied first before activating the state.
         */
        attr = tb[DEVLINK_PORT_FN_ATTR_STATE];
        if (attr)
                err = devlink_port_fn_state_set(port, attr, extack);

        if (!err)
                devlink_port_notify(port, DEVLINK_CMD_PORT_NEW);
        return err;
}

int devlink_nl_port_set_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink_port *devlink_port = info->user_ptr[1];
        int err;

        if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
                enum devlink_port_type port_type;

                port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
                err = devlink_port_type_set(devlink_port, port_type);
                if (err)
                        return err;
        }

        if (info->attrs[DEVLINK_ATTR_PORT_FUNCTION]) {
                struct nlattr *attr = info->attrs[DEVLINK_ATTR_PORT_FUNCTION];
                struct netlink_ext_ack *extack = info->extack;

                err = devlink_port_function_set(devlink_port, attr, extack);
                if (err)
                        return err;
        }

        return 0;
}

int devlink_nl_port_split_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink_port *devlink_port = info->user_ptr[1];
        struct devlink *devlink = info->user_ptr[0];
        u32 count;

        if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_SPLIT_COUNT))
                return -EINVAL;
        if (!devlink_port->ops->port_split)
                return -EOPNOTSUPP;

        count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);

        if (!devlink_port->attrs.splittable) {
                /* Split ports cannot be split. */
                if (devlink_port->attrs.split)
                        NL_SET_ERR_MSG(info->extack, "Port cannot be split further");
                else
                        NL_SET_ERR_MSG(info->extack, "Port cannot be split");
                return -EINVAL;
        }

        if (count < 2 || !is_power_of_2(count) || count > devlink_port->attrs.lanes) {
                NL_SET_ERR_MSG(info->extack, "Invalid split count");
                return -EINVAL;
        }

        return devlink_port->ops->port_split(devlink, devlink_port, count,
                                             info->extack);
}

int devlink_nl_port_unsplit_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink_port *devlink_port = info->user_ptr[1];
        struct devlink *devlink = info->user_ptr[0];

        if (!devlink_port->ops->port_unsplit)
                return -EOPNOTSUPP;
        return devlink_port->ops->port_unsplit(devlink, devlink_port, info->extack);
}

int devlink_nl_port_new_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct netlink_ext_ack *extack = info->extack;
        struct devlink_port_new_attrs new_attrs = {};
        struct devlink *devlink = info->user_ptr[0];
        struct devlink_port *devlink_port;
        struct sk_buff *msg;
        int err;

        if (!devlink->ops->port_new)
                return -EOPNOTSUPP;

        if (!info->attrs[DEVLINK_ATTR_PORT_FLAVOUR] ||
            !info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]) {
                NL_SET_ERR_MSG(extack, "Port flavour or PCI PF are not specified");
                return -EINVAL;
        }
        new_attrs.flavour = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_FLAVOUR]);
        new_attrs.pfnum =
                nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]);

        if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
                /* Port index of the new port being created by driver. */
                new_attrs.port_index =
                        nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
                new_attrs.port_index_valid = true;
        }
        if (info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]) {
                new_attrs.controller =
                        nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]);
                new_attrs.controller_valid = true;
        }
        if (new_attrs.flavour == DEVLINK_PORT_FLAVOUR_PCI_SF &&
            info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]) {
                new_attrs.sfnum = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]);
                new_attrs.sfnum_valid = true;
        }

        err = devlink->ops->port_new(devlink, &new_attrs,
                                     extack, &devlink_port);
        if (err)
                return err;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg) {
                err = -ENOMEM;
                goto err_out_port_del;
        }
        err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_PORT_NEW,
                                   info->snd_portid, info->snd_seq, 0, NULL);
        if (WARN_ON_ONCE(err))
                goto err_out_msg_free;
        err = genlmsg_reply(msg, info);
        if (err)
                goto err_out_port_del;
        return 0;

err_out_msg_free:
        nlmsg_free(msg);
err_out_port_del:
        devlink_port->ops->port_del(devlink, devlink_port, NULL);
        return err;
}

int devlink_nl_port_del_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct devlink_port *devlink_port = info->user_ptr[1];
        struct netlink_ext_ack *extack = info->extack;
        struct devlink *devlink = info->user_ptr[0];

        if (!devlink_port->ops->port_del)
                return -EOPNOTSUPP;

        return devlink_port->ops->port_del(devlink, devlink_port, extack);
}

static void devlink_port_type_warn(struct work_struct *work)
{
        struct devlink_port *port = container_of(to_delayed_work(work),
                                                 struct devlink_port,
                                                 type_warn_dw);
        dev_warn(port->devlink->dev, "Type was not set for devlink port.");
}

static bool devlink_port_type_should_warn(struct devlink_port *devlink_port)
{
        /* Ignore CPU and DSA flavours. */
        return devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_CPU &&
               devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_DSA &&
               devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_UNUSED;
}

#define DEVLINK_PORT_TYPE_WARN_TIMEOUT (HZ * 3600)

static void devlink_port_type_warn_schedule(struct devlink_port *devlink_port)
{
        if (!devlink_port_type_should_warn(devlink_port))
                return;
        /* Schedule a work to WARN in case driver does not set port
         * type within timeout.
         */
        schedule_delayed_work(&devlink_port->type_warn_dw,
                              DEVLINK_PORT_TYPE_WARN_TIMEOUT);
}

static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port)
{
        if (!devlink_port_type_should_warn(devlink_port))
                return;
        cancel_delayed_work_sync(&devlink_port->type_warn_dw);
}

/**
 * devlink_port_init() - Init devlink port
 *
 * @devlink: devlink
 * @devlink_port: devlink port
 *
 * Initialize essential stuff that is needed for functions
 * that may be called before devlink port registration.
 * Call to this function is optional and not needed
 * in case the driver does not use such functions.
 */
void devlink_port_init(struct devlink *devlink,
                       struct devlink_port *devlink_port)
{
        if (devlink_port->initialized)
                return;
        devlink_port->devlink = devlink;
        INIT_LIST_HEAD(&devlink_port->region_list);
        devlink_port->initialized = true;
}
EXPORT_SYMBOL_GPL(devlink_port_init);

/**
 * devlink_port_fini() - Deinitialize devlink port
 *
 * @devlink_port: devlink port
 *
 * Deinitialize essential stuff that is in use for functions
 * that may be called after devlink port unregistration.
 * Call to this function is optional and not needed
 * in case the driver does not use such functions.
 */
void devlink_port_fini(struct devlink_port *devlink_port)
{
        WARN_ON(!list_empty(&devlink_port->region_list));
}
EXPORT_SYMBOL_GPL(devlink_port_fini);

static const struct devlink_port_ops devlink_port_dummy_ops = {};

/**
 * devl_port_register_with_ops() - Register devlink port
 *
 * @devlink: devlink
 * @devlink_port: devlink port
 * @port_index: driver-specific numerical identifier of the port
 * @ops: port ops
 *
 * Register devlink port with provided port index. User can use
 * any indexing, even hw-related one. devlink_port structure
 * is convenient to be embedded inside user driver private structure.
 * Note that the caller should take care of zeroing the devlink_port
 * structure.
 */
int devl_port_register_with_ops(struct devlink *devlink,
                                struct devlink_port *devlink_port,
                                unsigned int port_index,
                                const struct devlink_port_ops *ops)
{
        int err;

        devl_assert_locked(devlink);

        ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);

        devlink_port_init(devlink, devlink_port);
        devlink_port->registered = true;
        devlink_port->index = port_index;
        devlink_port->ops = ops ? ops : &devlink_port_dummy_ops;
        spin_lock_init(&devlink_port->type_lock);
        INIT_LIST_HEAD(&devlink_port->reporter_list);
        err = xa_insert(&devlink->ports, port_index, devlink_port, GFP_KERNEL);
        if (err) {
                devlink_port->registered = false;
                return err;
        }

        INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn);
        devlink_port_type_warn_schedule(devlink_port);
        devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
        return 0;
}
EXPORT_SYMBOL_GPL(devl_port_register_with_ops);

/**
 *        devlink_port_register_with_ops - Register devlink port
 *
 *        @devlink: devlink
 *        @devlink_port: devlink port
 *        @port_index: driver-specific numerical identifier of the port
 *        @ops: port ops
 *
 *        Register devlink port with provided port index. User can use
 *        any indexing, even hw-related one. devlink_port structure
 *        is convenient to be embedded inside user driver private structure.
 *        Note that the caller should take care of zeroing the devlink_port
 *        structure.
 *
 *        Context: Takes and release devlink->lock <mutex>.
 */
int devlink_port_register_with_ops(struct devlink *devlink,
                                   struct devlink_port *devlink_port,
                                   unsigned int port_index,
                                   const struct devlink_port_ops *ops)
{
        int err;

        devl_lock(devlink);
        err = devl_port_register_with_ops(devlink, devlink_port,
                                          port_index, ops);
        devl_unlock(devlink);
        return err;
}
EXPORT_SYMBOL_GPL(devlink_port_register_with_ops);

/**
 * devl_port_unregister() - Unregister devlink port
 *
 * @devlink_port: devlink port
 */
void devl_port_unregister(struct devlink_port *devlink_port)
{
        lockdep_assert_held(&devlink_port->devlink->lock);
        WARN_ON(devlink_port->type != DEVLINK_PORT_TYPE_NOTSET);

        devlink_port_type_warn_cancel(devlink_port);
        devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
        xa_erase(&devlink_port->devlink->ports, devlink_port->index);
        WARN_ON(!list_empty(&devlink_port->reporter_list));
        devlink_port->registered = false;
}
EXPORT_SYMBOL_GPL(devl_port_unregister);

/**
 *        devlink_port_unregister - Unregister devlink port
 *
 *        @devlink_port: devlink port
 *
 *        Context: Takes and release devlink->lock <mutex>.
 */
void devlink_port_unregister(struct devlink_port *devlink_port)
{
        struct devlink *devlink = devlink_port->devlink;

        devl_lock(devlink);
        devl_port_unregister(devlink_port);
        devl_unlock(devlink);
}
EXPORT_SYMBOL_GPL(devlink_port_unregister);

static void devlink_port_type_netdev_checks(struct devlink_port *devlink_port,
                                            struct net_device *netdev)
{
        const struct net_device_ops *ops = netdev->netdev_ops;

        /* If driver registers devlink port, it should set devlink port
         * attributes accordingly so the compat functions are called
         * and the original ops are not used.
         */
        if (ops->ndo_get_phys_port_name) {
                /* Some drivers use the same set of ndos for netdevs
                 * that have devlink_port registered and also for
                 * those who don't. Make sure that ndo_get_phys_port_name
                 * returns -EOPNOTSUPP here in case it is defined.
                 * Warn if not.
                 */
                char name[IFNAMSIZ];
                int err;

                err = ops->ndo_get_phys_port_name(netdev, name, sizeof(name));
                WARN_ON(err != -EOPNOTSUPP);
        }
        if (ops->ndo_get_port_parent_id) {
                /* Some drivers use the same set of ndos for netdevs
                 * that have devlink_port registered and also for
                 * those who don't. Make sure that ndo_get_port_parent_id
                 * returns -EOPNOTSUPP here in case it is defined.
                 * Warn if not.
                 */
                struct netdev_phys_item_id ppid;
                int err;

                err = ops->ndo_get_port_parent_id(netdev, &ppid);
                WARN_ON(err != -EOPNOTSUPP);
        }
}

static void __devlink_port_type_set(struct devlink_port *devlink_port,
                                    enum devlink_port_type type,
                                    void *type_dev)
{
        struct net_device *netdev = type_dev;

        ASSERT_DEVLINK_PORT_REGISTERED(devlink_port);

        if (type == DEVLINK_PORT_TYPE_NOTSET) {
                devlink_port_type_warn_schedule(devlink_port);
        } else {
                devlink_port_type_warn_cancel(devlink_port);
                if (type == DEVLINK_PORT_TYPE_ETH && netdev)
                        devlink_port_type_netdev_checks(devlink_port, netdev);
        }

        spin_lock_bh(&devlink_port->type_lock);
        devlink_port->type = type;
        switch (type) {
        case DEVLINK_PORT_TYPE_ETH:
                devlink_port->type_eth.netdev = netdev;
                if (netdev) {
                        ASSERT_RTNL();
                        devlink_port->type_eth.ifindex = netdev->ifindex;
                        BUILD_BUG_ON(sizeof(devlink_port->type_eth.ifname) !=
                                     sizeof(netdev->name));
                        strcpy(devlink_port->type_eth.ifname, netdev->name);
                }
                break;
        case DEVLINK_PORT_TYPE_IB:
                devlink_port->type_ib.ibdev = type_dev;
                break;
        default:
                break;
        }
        spin_unlock_bh(&devlink_port->type_lock);
        devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
}

/**
 *        devlink_port_type_eth_set - Set port type to Ethernet
 *
 *        @devlink_port: devlink port
 *
 *        If driver is calling this, most likely it is doing something wrong.
 */
void devlink_port_type_eth_set(struct devlink_port *devlink_port)
{
        dev_warn(devlink_port->devlink->dev,
                 "devlink port type for port %d set to Ethernet without a software interface reference, device type not supported by the kernel?\n",
                 devlink_port->index);
        __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_ETH, NULL);
}
EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);

/**
 *        devlink_port_type_ib_set - Set port type to InfiniBand
 *
 *        @devlink_port: devlink port
 *        @ibdev: related IB device
 */
void devlink_port_type_ib_set(struct devlink_port *devlink_port,
                              struct ib_device *ibdev)
{
        __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_IB, ibdev);
}
EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);

/**
 *        devlink_port_type_clear - Clear port type
 *
 *        @devlink_port: devlink port
 *
 *        If driver is calling this for clearing Ethernet type, most likely
 *        it is doing something wrong.
 */
void devlink_port_type_clear(struct devlink_port *devlink_port)
{
        if (devlink_port->type == DEVLINK_PORT_TYPE_ETH)
                dev_warn(devlink_port->devlink->dev,
                         "devlink port type for port %d cleared without a software interface reference, device type not supported by the kernel?\n",
                         devlink_port->index);
        __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_NOTSET, NULL);
}
EXPORT_SYMBOL_GPL(devlink_port_type_clear);

int devlink_port_netdevice_event(struct notifier_block *nb,
                                 unsigned long event, void *ptr)
{
        struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
        struct devlink_port *devlink_port = netdev->devlink_port;
        struct devlink *devlink;

        if (!devlink_port)
                return NOTIFY_OK;
        devlink = devlink_port->devlink;

        switch (event) {
        case NETDEV_POST_INIT:
                /* Set the type but not netdev pointer. It is going to be set
                 * later on by NETDEV_REGISTER event. Happens once during
                 * netdevice register
                 */
                __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_ETH,
                                        NULL);
                break;
        case NETDEV_REGISTER:
        case NETDEV_CHANGENAME:
                if (devlink_net(devlink) != dev_net(netdev))
                        return NOTIFY_OK;
                /* Set the netdev on top of previously set type. Note this
                 * event happens also during net namespace change so here
                 * we take into account netdev pointer appearing in this
                 * namespace.
                 */
                __devlink_port_type_set(devlink_port, devlink_port->type,
                                        netdev);
                break;
        case NETDEV_UNREGISTER:
                if (devlink_net(devlink) != dev_net(netdev))
                        return NOTIFY_OK;
                /* Clear netdev pointer, but not the type. This event happens
                 * also during net namespace change so we need to clear
                 * pointer to netdev that is going to another net namespace.
                 */
                __devlink_port_type_set(devlink_port, devlink_port->type,
                                        NULL);
                break;
        case NETDEV_PRE_UNINIT:
                /* Clear the type and the netdev pointer. Happens one during
                 * netdevice unregister.
                 */
                __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_NOTSET,
                                        NULL);
                break;
        }

        return NOTIFY_OK;
}

static void __devlink_port_attrs_set(struct devlink_port *devlink_port,
                                     enum devlink_port_flavour flavour)
{
        struct devlink_port_attrs *attrs = &devlink_port->attrs;

        devlink_port->attrs_set = true;
        attrs->flavour = flavour;
        if (attrs->switch_id.id_len) {
                devlink_port->switch_port = true;
                if (WARN_ON(attrs->switch_id.id_len > MAX_PHYS_ITEM_ID_LEN))
                        attrs->switch_id.id_len = MAX_PHYS_ITEM_ID_LEN;
        } else {
                devlink_port->switch_port = false;
        }
}

/**
 *        devlink_port_attrs_set - Set port attributes
 *
 *        @devlink_port: devlink port
 *        @attrs: devlink port attrs
 */
void devlink_port_attrs_set(struct devlink_port *devlink_port,
                            const struct devlink_port_attrs *attrs)
{
        ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
        WARN_ON(attrs->splittable && attrs->split);

        devlink_port->attrs = *attrs;
        __devlink_port_attrs_set(devlink_port, attrs->flavour);
}
EXPORT_SYMBOL_GPL(devlink_port_attrs_set);

/**
 *        devlink_port_attrs_pci_pf_set - Set PCI PF port attributes
 *
 *        @devlink_port: devlink port
 *        @controller: associated controller number for the devlink port instance
 *        @pf: associated PCI function number for the devlink port instance
 *        @external: indicates if the port is for an external controller
 */
void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
                                   u16 pf, bool external)
{
        struct devlink_port_attrs *attrs = &devlink_port->attrs;

        ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);

        __devlink_port_attrs_set(devlink_port, DEVLINK_PORT_FLAVOUR_PCI_PF);
        attrs->pci_pf.controller = controller;
        attrs->pci_pf.pf = pf;
        attrs->pci_pf.external = external;
}
EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_pf_set);

/**
 *        devlink_port_attrs_pci_vf_set - Set PCI VF port attributes
 *
 *        @devlink_port: devlink port
 *        @controller: associated controller number for the devlink port instance
 *        @pf: associated PCI function number for the devlink port instance
 *        @vf: associated PCI VF number of a PF for the devlink port instance;
 *             VF number starts from 0 for the first PCI virtual function
 *        @external: indicates if the port is for an external controller
 */
void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
                                   u16 pf, u16 vf, bool external)
{
        struct devlink_port_attrs *attrs = &devlink_port->attrs;

        ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);

        __devlink_port_attrs_set(devlink_port, DEVLINK_PORT_FLAVOUR_PCI_VF);
        attrs->pci_vf.controller = controller;
        attrs->pci_vf.pf = pf;
        attrs->pci_vf.vf = vf;
        attrs->pci_vf.external = external;
}
EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_vf_set);

/**
 *        devlink_port_attrs_pci_sf_set - Set PCI SF port attributes
 *
 *        @devlink_port: devlink port
 *        @controller: associated controller number for the devlink port instance
 *        @pf: associated PCI function number for the devlink port instance
 *        @sf: associated SF number of a PF for the devlink port instance
 *        @external: indicates if the port is for an external controller
 */
void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 controller,
                                   u16 pf, u32 sf, bool external)
{
        struct devlink_port_attrs *attrs = &devlink_port->attrs;

        ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);

        __devlink_port_attrs_set(devlink_port, DEVLINK_PORT_FLAVOUR_PCI_SF);
        attrs->pci_sf.controller = controller;
        attrs->pci_sf.pf = pf;
        attrs->pci_sf.sf = sf;
        attrs->pci_sf.external = external;
}
EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_sf_set);

static void devlink_port_rel_notify_cb(struct devlink *devlink, u32 port_index)
{
        struct devlink_port *devlink_port;

        devlink_port = devlink_port_get_by_index(devlink, port_index);
        if (!devlink_port)
                return;
        devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
}

static void devlink_port_rel_cleanup_cb(struct devlink *devlink, u32 port_index,
                                        u32 rel_index)
{
        struct devlink_port *devlink_port;

        devlink_port = devlink_port_get_by_index(devlink, port_index);
        if (devlink_port && devlink_port->rel_index == rel_index)
                devlink_port->rel_index = 0;
}

/**
 * devl_port_fn_devlink_set - Attach peer devlink
 *                              instance to port function.
 * @devlink_port: devlink port
 * @fn_devlink: devlink instance to attach
 */
int devl_port_fn_devlink_set(struct devlink_port *devlink_port,
                             struct devlink *fn_devlink)
{
        ASSERT_DEVLINK_PORT_REGISTERED(devlink_port);

        if (WARN_ON(devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_SF ||
                    devlink_port->attrs.pci_sf.external))
                return -EINVAL;

        return devlink_rel_nested_in_add(&devlink_port->rel_index,
                                         devlink_port->devlink->index,
                                         devlink_port->index,
                                         devlink_port_rel_notify_cb,
                                         devlink_port_rel_cleanup_cb,
                                         fn_devlink);
}
EXPORT_SYMBOL_GPL(devl_port_fn_devlink_set);

/**
 *        devlink_port_linecard_set - Link port with a linecard
 *
 *        @devlink_port: devlink port
 *        @linecard: devlink linecard
 */
void devlink_port_linecard_set(struct devlink_port *devlink_port,
                               struct devlink_linecard *linecard)
{
        ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);

        devlink_port->linecard = linecard;
}
EXPORT_SYMBOL_GPL(devlink_port_linecard_set);

static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
                                             char *name, size_t len)
{
        struct devlink_port_attrs *attrs = &devlink_port->attrs;
        int n = 0;

        if (!devlink_port->attrs_set || devlink_port->attrs.no_phys_port_name)
                return -EOPNOTSUPP;

        switch (attrs->flavour) {
        case DEVLINK_PORT_FLAVOUR_PHYSICAL:
                if (devlink_port->linecard)
                        n = snprintf(name, len, "l%u",
                                     devlink_linecard_index(devlink_port->linecard));
                if (n < len)
                        n += snprintf(name + n, len - n, "p%u",
                                      attrs->phys.port_number);
                if (n < len && attrs->split)
                        n += snprintf(name + n, len - n, "s%u",
                                      attrs->phys.split_subport_number);
                break;
        case DEVLINK_PORT_FLAVOUR_CPU:
        case DEVLINK_PORT_FLAVOUR_DSA:
        case DEVLINK_PORT_FLAVOUR_UNUSED:
                /* As CPU and DSA ports do not have a netdevice associated
                 * case should not ever happen.
                 */
                WARN_ON(1);
                return -EINVAL;
        case DEVLINK_PORT_FLAVOUR_PCI_PF:
                if (attrs->pci_pf.external) {
                        n = snprintf(name, len, "c%u", attrs->pci_pf.controller);
                        if (n >= len)
                                return -EINVAL;
                        len -= n;
                        name += n;
                }
                n = snprintf(name, len, "pf%u", attrs->pci_pf.pf);
                break;
        case DEVLINK_PORT_FLAVOUR_PCI_VF:
                if (attrs->pci_vf.external) {
                        n = snprintf(name, len, "c%u", attrs->pci_vf.controller);
                        if (n >= len)
                                return -EINVAL;
                        len -= n;
                        name += n;
                }
                n = snprintf(name, len, "pf%uvf%u",
                             attrs->pci_vf.pf, attrs->pci_vf.vf);
                break;
        case DEVLINK_PORT_FLAVOUR_PCI_SF:
                if (attrs->pci_sf.external) {
                        n = snprintf(name, len, "c%u", attrs->pci_sf.controller);
                        if (n >= len)
                                return -EINVAL;
                        len -= n;
                        name += n;
                }
                n = snprintf(name, len, "pf%usf%u", attrs->pci_sf.pf,
                             attrs->pci_sf.sf);
                break;
        case DEVLINK_PORT_FLAVOUR_VIRTUAL:
                return -EOPNOTSUPP;
        }

        if (n >= len)
                return -EINVAL;

        return 0;
}

int devlink_compat_phys_port_name_get(struct net_device *dev,
                                      char *name, size_t len)
{
        struct devlink_port *devlink_port;

        /* RTNL mutex is held here which ensures that devlink_port
         * instance cannot disappear in the middle. No need to take
         * any devlink lock as only permanent values are accessed.
         */
        ASSERT_RTNL();

        devlink_port = dev->devlink_port;
        if (!devlink_port)
                return -EOPNOTSUPP;

        return __devlink_port_phys_port_name_get(devlink_port, name, len);
}

int devlink_compat_switch_id_get(struct net_device *dev,
                                 struct netdev_phys_item_id *ppid)
{
        struct devlink_port *devlink_port;

        /* Caller must hold RTNL mutex or reference to dev, which ensures that
         * devlink_port instance cannot disappear in the middle. No need to take
         * any devlink lock as only permanent values are accessed.
         */
        devlink_port = dev->devlink_port;
        if (!devlink_port || !devlink_port->switch_port)
                return -EOPNOTSUPP;

        memcpy(ppid, &devlink_port->attrs.switch_id, sizeof(*ppid));

        return 0;
}




































































































































































  857 
































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef _LINUX_OF_H
#define _LINUX_OF_H
/*
 * Definitions for talking to the Open Firmware PROM on
 * Power Macintosh and other computers.
 *
 * Copyright (C) 1996-2005 Paul Mackerras.
 *
 * Updates for PPC64 by Peter Bergner & David Engebretsen, IBM Corp.
 * Updates for SPARC64 by David S. Miller
 * Derived from PowerPC and Sparc prom.h files by Stephen Rothwell, IBM Corp.
 */
#include <linux/types.h>
#include <linux/bitops.h>
#include <linux/cleanup.h>
#include <linux/errno.h>
#include <linux/kobject.h>
#include <linux/mod_devicetable.h>
#include <linux/property.h>
#include <linux/list.h>

#include <asm/byteorder.h>

typedef u32 phandle;
typedef u32 ihandle;

struct property {
        char        *name;
        int        length;
        void        *value;
        struct property *next;
#if defined(CONFIG_OF_DYNAMIC) || defined(CONFIG_SPARC)
        unsigned long _flags;
#endif
#if defined(CONFIG_OF_PROMTREE)
        unsigned int unique_id;
#endif
#if defined(CONFIG_OF_KOBJ)
        struct bin_attribute attr;
#endif
};

#if defined(CONFIG_SPARC)
struct of_irq_controller;
#endif

struct device_node {
        const char *name;
        phandle phandle;
        const char *full_name;
        struct fwnode_handle fwnode;

        struct        property *properties;
        struct        property *deadprops;        /* removed properties */
        struct        device_node *parent;
        struct        device_node *child;
        struct        device_node *sibling;
#if defined(CONFIG_OF_KOBJ)
        struct        kobject kobj;
#endif
        unsigned long _flags;
        void        *data;
#if defined(CONFIG_SPARC)
        unsigned int unique_id;
        struct of_irq_controller *irq_trans;
#endif
};

#define MAX_PHANDLE_ARGS NR_FWNODE_REFERENCE_ARGS
struct of_phandle_args {
        struct device_node *np;
        int args_count;
        uint32_t args[MAX_PHANDLE_ARGS];
};

struct of_phandle_iterator {
        /* Common iterator information */
        const char *cells_name;
        int cell_count;
        const struct device_node *parent;

        /* List size information */
        const __be32 *list_end;
        const __be32 *phandle_end;

        /* Current position state */
        const __be32 *cur;
        uint32_t cur_count;
        phandle phandle;
        struct device_node *node;
};

struct of_reconfig_data {
        struct device_node        *dn;
        struct property                *prop;
        struct property                *old_prop;
};

extern const struct kobj_type of_node_ktype;
extern const struct fwnode_operations of_fwnode_ops;

/**
 * of_node_init - initialize a devicetree node
 * @node: Pointer to device node that has been created by kzalloc()
 *
 * On return the device_node refcount is set to one.  Use of_node_put()
 * on @node when done to free the memory allocated for it.  If the node
 * is NOT a dynamic node the memory will not be freed. The decision of
 * whether to free the memory will be done by node->release(), which is
 * of_node_release().
 */
static inline void of_node_init(struct device_node *node)
{
#if defined(CONFIG_OF_KOBJ)
        kobject_init(&node->kobj, &of_node_ktype);
#endif
        fwnode_init(&node->fwnode, &of_fwnode_ops);
}

#if defined(CONFIG_OF_KOBJ)
#define of_node_kobj(n) (&(n)->kobj)
#else
#define of_node_kobj(n) NULL
#endif

#ifdef CONFIG_OF_DYNAMIC
extern struct device_node *of_node_get(struct device_node *node);
extern void of_node_put(struct device_node *node);
#else /* CONFIG_OF_DYNAMIC */
/* Dummy ref counting routines - to be implemented later */
static inline struct device_node *of_node_get(struct device_node *node)
{
        return node;
}
static inline void of_node_put(struct device_node *node) { }
#endif /* !CONFIG_OF_DYNAMIC */
DEFINE_FREE(device_node, struct device_node *, if (_T) of_node_put(_T))

/* Pointer for first entry in chain of all nodes. */
extern struct device_node *of_root;
extern struct device_node *of_chosen;
extern struct device_node *of_aliases;
extern struct device_node *of_stdout;

/*
 * struct device_node flag descriptions
 * (need to be visible even when !CONFIG_OF)
 */
#define OF_DYNAMIC                1 /* (and properties) allocated via kmalloc */
#define OF_DETACHED                2 /* detached from the device tree */
#define OF_POPULATED                3 /* device already created */
#define OF_POPULATED_BUS        4 /* platform bus created for children */
#define OF_OVERLAY                5 /* allocated for an overlay */
#define OF_OVERLAY_FREE_CSET        6 /* in overlay cset being freed */

#define OF_BAD_ADDR        ((u64)-1)

#ifdef CONFIG_OF
void of_core_init(void);

static inline bool is_of_node(const struct fwnode_handle *fwnode)
{
        return !IS_ERR_OR_NULL(fwnode) && fwnode->ops == &of_fwnode_ops;
}

#define to_of_node(__fwnode)                                                \
        ({                                                                \
                typeof(__fwnode) __to_of_node_fwnode = (__fwnode);        \
                                                                        \
                is_of_node(__to_of_node_fwnode) ?                        \
                        container_of(__to_of_node_fwnode,                \
                                     struct device_node, fwnode) :        \
                        NULL;                                                \
        })

#define of_fwnode_handle(node)                                                \
        ({                                                                \
                typeof(node) __of_fwnode_handle_node = (node);                \
                                                                        \
                __of_fwnode_handle_node ?                                \
                        &__of_fwnode_handle_node->fwnode : NULL;        \
        })

static inline bool of_node_is_root(const struct device_node *node)
{
        return node && (node->parent == NULL);
}

static inline int of_node_check_flag(const struct device_node *n, unsigned long flag)
{
        return test_bit(flag, &n->_flags);
}

static inline int of_node_test_and_set_flag(struct device_node *n,
                                            unsigned long flag)
{
        return test_and_set_bit(flag, &n->_flags);
}

static inline void of_node_set_flag(struct device_node *n, unsigned long flag)
{
        set_bit(flag, &n->_flags);
}

static inline void of_node_clear_flag(struct device_node *n, unsigned long flag)
{
        clear_bit(flag, &n->_flags);
}

#if defined(CONFIG_OF_DYNAMIC) || defined(CONFIG_SPARC)
static inline int of_property_check_flag(const struct property *p, unsigned long flag)
{
        return test_bit(flag, &p->_flags);
}

static inline void of_property_set_flag(struct property *p, unsigned long flag)
{
        set_bit(flag, &p->_flags);
}

static inline void of_property_clear_flag(struct property *p, unsigned long flag)
{
        clear_bit(flag, &p->_flags);
}
#endif

extern struct device_node *__of_find_all_nodes(struct device_node *prev);
extern struct device_node *of_find_all_nodes(struct device_node *prev);

/*
 * OF address retrieval & translation
 */

/* Helper to read a big number; size is in cells (not bytes) */
static inline u64 of_read_number(const __be32 *cell, int size)
{
        u64 r = 0;
        for (; size--; cell++)
                r = (r << 32) | be32_to_cpu(*cell);
        return r;
}

/* Like of_read_number, but we want an unsigned long result */
static inline unsigned long of_read_ulong(const __be32 *cell, int size)
{
        /* toss away upper bits if unsigned long is smaller than u64 */
        return of_read_number(cell, size);
}

#if defined(CONFIG_SPARC)
#include <asm/prom.h>
#endif

#define OF_IS_DYNAMIC(x) test_bit(OF_DYNAMIC, &x->_flags)
#define OF_MARK_DYNAMIC(x) set_bit(OF_DYNAMIC, &x->_flags)

extern bool of_node_name_eq(const struct device_node *np, const char *name);
extern bool of_node_name_prefix(const struct device_node *np, const char *prefix);

static inline const char *of_node_full_name(const struct device_node *np)
{
        return np ? np->full_name : "<no-node>";
}

#define for_each_of_allnodes_from(from, dn) \
        for (dn = __of_find_all_nodes(from); dn; dn = __of_find_all_nodes(dn))
#define for_each_of_allnodes(dn) for_each_of_allnodes_from(NULL, dn)
extern struct device_node *of_find_node_by_name(struct device_node *from,
        const char *name);
extern struct device_node *of_find_node_by_type(struct device_node *from,
        const char *type);
extern struct device_node *of_find_compatible_node(struct device_node *from,
        const char *type, const char *compat);
extern struct device_node *of_find_matching_node_and_match(
        struct device_node *from,
        const struct of_device_id *matches,
        const struct of_device_id **match);

extern struct device_node *of_find_node_opts_by_path(const char *path,
        const char **opts);
static inline struct device_node *of_find_node_by_path(const char *path)
{
        return of_find_node_opts_by_path(path, NULL);
}

extern struct device_node *of_find_node_by_phandle(phandle handle);
extern struct device_node *of_get_parent(const struct device_node *node);
extern struct device_node *of_get_next_parent(struct device_node *node);
extern struct device_node *of_get_next_child(const struct device_node *node,
                                             struct device_node *prev);
extern struct device_node *of_get_next_child_with_prefix(const struct device_node *node,
                                                         struct device_node *prev,
                                                         const char *prefix);
extern struct device_node *of_get_next_available_child(
        const struct device_node *node, struct device_node *prev);
extern struct device_node *of_get_next_reserved_child(
        const struct device_node *node, struct device_node *prev);

extern struct device_node *of_get_compatible_child(const struct device_node *parent,
                                        const char *compatible);
extern struct device_node *of_get_child_by_name(const struct device_node *node,
                                        const char *name);
extern struct device_node *of_get_available_child_by_name(const struct device_node *node,
                                                          const char *name);

/* cache lookup */
extern struct device_node *of_find_next_cache_node(const struct device_node *);
extern int of_find_last_cache_level(unsigned int cpu);
extern struct device_node *of_find_node_with_property(
        struct device_node *from, const char *prop_name);

extern struct property *of_find_property(const struct device_node *np,
                                         const char *name,
                                         int *lenp);
extern bool of_property_read_bool(const struct device_node *np, const char *propname);
extern int of_property_count_elems_of_size(const struct device_node *np,
                                const char *propname, int elem_size);
extern int of_property_read_u16_index(const struct device_node *np,
                                       const char *propname,
                                       u32 index, u16 *out_value);
extern int of_property_read_u32_index(const struct device_node *np,
                                       const char *propname,
                                       u32 index, u32 *out_value);
extern int of_property_read_u64_index(const struct device_node *np,
                                       const char *propname,
                                       u32 index, u64 *out_value);
extern int of_property_read_variable_u8_array(const struct device_node *np,
                                        const char *propname, u8 *out_values,
                                        size_t sz_min, size_t sz_max);
extern int of_property_read_variable_u16_array(const struct device_node *np,
                                        const char *propname, u16 *out_values,
                                        size_t sz_min, size_t sz_max);
extern int of_property_read_variable_u32_array(const struct device_node *np,
                                        const char *propname,
                                        u32 *out_values,
                                        size_t sz_min,
                                        size_t sz_max);
extern int of_property_read_u64(const struct device_node *np,
                                const char *propname, u64 *out_value);
extern int of_property_read_variable_u64_array(const struct device_node *np,
                                        const char *propname,
                                        u64 *out_values,
                                        size_t sz_min,
                                        size_t sz_max);

extern int of_property_read_string(const struct device_node *np,
                                   const char *propname,
                                   const char **out_string);
extern int of_property_match_string(const struct device_node *np,
                                    const char *propname,
                                    const char *string);
extern int of_property_read_string_helper(const struct device_node *np,
                                              const char *propname,
                                              const char **out_strs, size_t sz, int index);
extern int of_device_is_compatible(const struct device_node *device,
                                   const char *);
extern int of_device_compatible_match(const struct device_node *device,
                                      const char *const *compat);
extern bool of_device_is_available(const struct device_node *device);
extern bool of_device_is_big_endian(const struct device_node *device);
extern const void *of_get_property(const struct device_node *node,
                                const char *name,
                                int *lenp);
extern struct device_node *of_get_cpu_node(int cpu, unsigned int *thread);
extern struct device_node *of_cpu_device_node_get(int cpu);
extern int of_cpu_node_to_id(struct device_node *np);
extern struct device_node *of_get_next_cpu_node(struct device_node *prev);
extern struct device_node *of_get_cpu_state_node(const struct device_node *cpu_node,
                                                 int index);
extern u64 of_get_cpu_hwid(struct device_node *cpun, unsigned int thread);

extern int of_n_addr_cells(struct device_node *np);
extern int of_n_size_cells(struct device_node *np);
extern const struct of_device_id *of_match_node(
        const struct of_device_id *matches, const struct device_node *node);
extern const void *of_device_get_match_data(const struct device *dev);
extern int of_alias_from_compatible(const struct device_node *node, char *alias,
                                    int len);
extern void of_print_phandle_args(const char *msg, const struct of_phandle_args *args);
extern int __of_parse_phandle_with_args(const struct device_node *np,
        const char *list_name, const char *cells_name, int cell_count,
        int index, struct of_phandle_args *out_args);
extern int of_parse_phandle_with_args_map(const struct device_node *np,
        const char *list_name, const char *stem_name, int index,
        struct of_phandle_args *out_args);
extern int of_count_phandle_with_args(const struct device_node *np,
        const char *list_name, const char *cells_name);

/* module functions */
extern ssize_t of_modalias(const struct device_node *np, char *str, ssize_t len);
extern int of_request_module(const struct device_node *np);

/* phandle iterator functions */
extern int of_phandle_iterator_init(struct of_phandle_iterator *it,
                                    const struct device_node *np,
                                    const char *list_name,
                                    const char *cells_name,
                                    int cell_count);

extern int of_phandle_iterator_next(struct of_phandle_iterator *it);
extern int of_phandle_iterator_args(struct of_phandle_iterator *it,
                                    uint32_t *args,
                                    int size);

extern int of_alias_get_id(const struct device_node *np, const char *stem);
extern int of_alias_get_highest_id(const char *stem);

bool of_machine_compatible_match(const char *const *compats);

/**
 * of_machine_is_compatible - Test root of device tree for a given compatible value
 * @compat: compatible string to look for in root node's compatible property.
 *
 * Return: true if the root node has the given value in its compatible property.
 */
static inline bool of_machine_is_compatible(const char *compat)
{
        const char *compats[] = { compat, NULL };

        return of_machine_compatible_match(compats);
}

extern int of_add_property(struct device_node *np, struct property *prop);
extern int of_remove_property(struct device_node *np, struct property *prop);
extern int of_update_property(struct device_node *np, struct property *newprop);

/* For updating the device tree at runtime */
#define OF_RECONFIG_ATTACH_NODE                0x0001
#define OF_RECONFIG_DETACH_NODE                0x0002
#define OF_RECONFIG_ADD_PROPERTY        0x0003
#define OF_RECONFIG_REMOVE_PROPERTY        0x0004
#define OF_RECONFIG_UPDATE_PROPERTY        0x0005

extern int of_attach_node(struct device_node *);
extern int of_detach_node(struct device_node *);

#define of_match_ptr(_ptr)        (_ptr)

/*
 * u32 u;
 *
 * of_property_for_each_u32(np, "propname", u)
 *         printk("U32 value: %x\n", u);
 */
const __be32 *of_prop_next_u32(const struct property *prop, const __be32 *cur,
                               u32 *pu);
/*
 * struct property *prop;
 * const char *s;
 *
 * of_property_for_each_string(np, "propname", prop, s)
 *         printk("String value: %s\n", s);
 */
const char *of_prop_next_string(const struct property *prop, const char *cur);

bool of_console_check(const struct device_node *dn, char *name, int index);

int of_map_id(const struct device_node *np, u32 id,
               const char *map_name, const char *map_mask_name,
               struct device_node **target, u32 *id_out);

phys_addr_t of_dma_get_max_cpu_address(struct device_node *np);

struct kimage;
void *of_kexec_alloc_and_setup_fdt(const struct kimage *image,
                                   unsigned long initrd_load_addr,
                                   unsigned long initrd_len,
                                   const char *cmdline, size_t extra_fdt_size);
#else /* CONFIG_OF */

static inline void of_core_init(void)
{
}

static inline bool is_of_node(const struct fwnode_handle *fwnode)
{
        return false;
}

static inline struct device_node *to_of_node(const struct fwnode_handle *fwnode)
{
        return NULL;
}

static inline bool of_node_name_eq(const struct device_node *np, const char *name)
{
        return false;
}

static inline bool of_node_name_prefix(const struct device_node *np, const char *prefix)
{
        return false;
}

static inline const char* of_node_full_name(const struct device_node *np)
{
        return "<no-node>";
}

static inline struct device_node *of_find_node_by_name(struct device_node *from,
        const char *name)
{
        return NULL;
}

static inline struct device_node *of_find_node_by_type(struct device_node *from,
        const char *type)
{
        return NULL;
}

static inline struct device_node *of_find_matching_node_and_match(
        struct device_node *from,
        const struct of_device_id *matches,
        const struct of_device_id **match)
{
        return NULL;
}

static inline struct device_node *of_find_node_by_path(const char *path)
{
        return NULL;
}

static inline struct device_node *of_find_node_opts_by_path(const char *path,
        const char **opts)
{
        return NULL;
}

static inline struct device_node *of_find_node_by_phandle(phandle handle)
{
        return NULL;
}

static inline struct device_node *of_get_parent(const struct device_node *node)
{
        return NULL;
}

static inline struct device_node *of_get_next_parent(struct device_node *node)
{
        return NULL;
}

static inline struct device_node *of_get_next_child(
        const struct device_node *node, struct device_node *prev)
{
        return NULL;
}

static inline struct device_node *of_get_next_child_with_prefix(
        const struct device_node *node, struct device_node *prev,
        const char *prefix)
{
        return NULL;
}

static inline struct device_node *of_get_next_available_child(
        const struct device_node *node, struct device_node *prev)
{
        return NULL;
}

static inline struct device_node *of_get_next_reserved_child(
        const struct device_node *node, struct device_node *prev)
{
        return NULL;
}

static inline struct device_node *of_find_node_with_property(
        struct device_node *from, const char *prop_name)
{
        return NULL;
}

#define of_fwnode_handle(node) NULL

static inline struct device_node *of_get_compatible_child(const struct device_node *parent,
                                        const char *compatible)
{
        return NULL;
}

static inline struct device_node *of_get_child_by_name(
                                        const struct device_node *node,
                                        const char *name)
{
        return NULL;
}

static inline struct device_node *of_get_available_child_by_name(
                                        const struct device_node *node,
                                        const char *name)
{
        return NULL;
}

static inline int of_device_is_compatible(const struct device_node *device,
                                          const char *name)
{
        return 0;
}

static inline  int of_device_compatible_match(const struct device_node *device,
                                              const char *const *compat)
{
        return 0;
}

static inline bool of_device_is_available(const struct device_node *device)
{
        return false;
}

static inline bool of_device_is_big_endian(const struct device_node *device)
{
        return false;
}

static inline struct property *of_find_property(const struct device_node *np,
                                                const char *name,
                                                int *lenp)
{
        return NULL;
}

static inline struct device_node *of_find_compatible_node(
                                                struct device_node *from,
                                                const char *type,
                                                const char *compat)
{
        return NULL;
}

static inline bool of_property_read_bool(const struct device_node *np,
                                        const char *propname)
{
        return false;
}

static inline int of_property_count_elems_of_size(const struct device_node *np,
                        const char *propname, int elem_size)
{
        return -ENOSYS;
}

static inline int of_property_read_u16_index(const struct device_node *np,
                        const char *propname, u32 index, u16 *out_value)
{
        return -ENOSYS;
}

static inline int of_property_read_u32_index(const struct device_node *np,
                        const char *propname, u32 index, u32 *out_value)
{
        return -ENOSYS;
}

static inline int of_property_read_u64_index(const struct device_node *np,
                        const char *propname, u32 index, u64 *out_value)
{
        return -ENOSYS;
}

static inline const void *of_get_property(const struct device_node *node,
                                const char *name,
                                int *lenp)
{
        return NULL;
}

static inline struct device_node *of_get_cpu_node(int cpu,
                                        unsigned int *thread)
{
        return NULL;
}

static inline struct device_node *of_cpu_device_node_get(int cpu)
{
        return NULL;
}

static inline int of_cpu_node_to_id(struct device_node *np)
{
        return -ENODEV;
}

static inline struct device_node *of_get_next_cpu_node(struct device_node *prev)
{
        return NULL;
}

static inline struct device_node *of_get_cpu_state_node(struct device_node *cpu_node,
                                        int index)
{
        return NULL;
}

static inline int of_n_addr_cells(struct device_node *np)
{
        return 0;

}
static inline int of_n_size_cells(struct device_node *np)
{
        return 0;
}

static inline int of_property_read_variable_u8_array(const struct device_node *np,
                                        const char *propname, u8 *out_values,
                                        size_t sz_min, size_t sz_max)
{
        return -ENOSYS;
}

static inline int of_property_read_variable_u16_array(const struct device_node *np,
                                        const char *propname, u16 *out_values,
                                        size_t sz_min, size_t sz_max)
{
        return -ENOSYS;
}

static inline int of_property_read_variable_u32_array(const struct device_node *np,
                                        const char *propname,
                                        u32 *out_values,
                                        size_t sz_min,
                                        size_t sz_max)
{
        return -ENOSYS;
}

static inline int of_property_read_u64(const struct device_node *np,
                                       const char *propname, u64 *out_value)
{
        return -ENOSYS;
}

static inline int of_property_read_variable_u64_array(const struct device_node *np,
                                        const char *propname,
                                        u64 *out_values,
                                        size_t sz_min,
                                        size_t sz_max)
{
        return -ENOSYS;
}

static inline int of_property_read_string(const struct device_node *np,
                                          const char *propname,
                                          const char **out_string)
{
        return -ENOSYS;
}

static inline int of_property_match_string(const struct device_node *np,
                                           const char *propname,
                                           const char *string)
{
        return -ENOSYS;
}

static inline int of_property_read_string_helper(const struct device_node *np,
                                                 const char *propname,
                                                 const char **out_strs, size_t sz, int index)
{
        return -ENOSYS;
}

static inline int __of_parse_phandle_with_args(const struct device_node *np,
                                               const char *list_name,
                                               const char *cells_name,
                                               int cell_count,
                                               int index,
                                               struct of_phandle_args *out_args)
{
        return -ENOSYS;
}

static inline int of_parse_phandle_with_args_map(const struct device_node *np,
                                                 const char *list_name,
                                                 const char *stem_name,
                                                 int index,
                                                 struct of_phandle_args *out_args)
{
        return -ENOSYS;
}

static inline int of_count_phandle_with_args(const struct device_node *np,
                                             const char *list_name,
                                             const char *cells_name)
{
        return -ENOSYS;
}

static inline ssize_t of_modalias(const struct device_node *np, char *str,
                                  ssize_t len)
{
        return -ENODEV;
}

static inline int of_request_module(const struct device_node *np)
{
        return -ENODEV;
}

static inline int of_phandle_iterator_init(struct of_phandle_iterator *it,
                                           const struct device_node *np,
                                           const char *list_name,
                                           const char *cells_name,
                                           int cell_count)
{
        return -ENOSYS;
}

static inline int of_phandle_iterator_next(struct of_phandle_iterator *it)
{
        return -ENOSYS;
}

static inline int of_phandle_iterator_args(struct of_phandle_iterator *it,
                                           uint32_t *args,
                                           int size)
{
        return 0;
}

static inline int of_alias_get_id(struct device_node *np, const char *stem)
{
        return -ENOSYS;
}

static inline int of_alias_get_highest_id(const char *stem)
{
        return -ENOSYS;
}

static inline int of_machine_is_compatible(const char *compat)
{
        return 0;
}

static inline int of_add_property(struct device_node *np, struct property *prop)
{
        return 0;
}

static inline int of_remove_property(struct device_node *np, struct property *prop)
{
        return 0;
}

static inline bool of_machine_compatible_match(const char *const *compats)
{
        return false;
}

static inline bool of_console_check(const struct device_node *dn, const char *name, int index)
{
        return false;
}

static inline const __be32 *of_prop_next_u32(const struct property *prop,
                const __be32 *cur, u32 *pu)
{
        return NULL;
}

static inline const char *of_prop_next_string(const struct property *prop,
                const char *cur)
{
        return NULL;
}

static inline int of_node_check_flag(struct device_node *n, unsigned long flag)
{
        return 0;
}

static inline int of_node_test_and_set_flag(struct device_node *n,
                                            unsigned long flag)
{
        return 0;
}

static inline void of_node_set_flag(struct device_node *n, unsigned long flag)
{
}

static inline void of_node_clear_flag(struct device_node *n, unsigned long flag)
{
}

static inline int of_property_check_flag(const struct property *p,
                                         unsigned long flag)
{
        return 0;
}

static inline void of_property_set_flag(struct property *p, unsigned long flag)
{
}

static inline void of_property_clear_flag(struct property *p, unsigned long flag)
{
}

static inline int of_map_id(const struct device_node *np, u32 id,
                             const char *map_name, const char *map_mask_name,
                             struct device_node **target, u32 *id_out)
{
        return -EINVAL;
}

static inline phys_addr_t of_dma_get_max_cpu_address(struct device_node *np)
{
        return PHYS_ADDR_MAX;
}

static inline const void *of_device_get_match_data(const struct device *dev)
{
        return NULL;
}

#define of_match_ptr(_ptr)        NULL
#define of_match_node(_matches, _node)        NULL
#endif /* CONFIG_OF */

/* Default string compare functions, Allow arch asm/prom.h to override */
#if !defined(of_compat_cmp)
#define of_compat_cmp(s1, s2, l)        strcasecmp((s1), (s2))
#define of_prop_cmp(s1, s2)                strcmp((s1), (s2))
#define of_node_cmp(s1, s2)                strcasecmp((s1), (s2))
#endif

#define for_each_property_of_node(dn, pp) \
        for (pp = dn->properties; pp != NULL; pp = pp->next)

#if defined(CONFIG_OF) && defined(CONFIG_NUMA)
extern int of_node_to_nid(struct device_node *np);
#else
static inline int of_node_to_nid(struct device_node *device)
{
        return NUMA_NO_NODE;
}
#endif

#ifdef CONFIG_OF_NUMA
extern int of_numa_init(void);
#else
static inline int of_numa_init(void)
{
        return -ENOSYS;
}
#endif

static inline struct device_node *of_find_matching_node(
        struct device_node *from,
        const struct of_device_id *matches)
{
        return of_find_matching_node_and_match(from, matches, NULL);
}

static inline const char *of_node_get_device_type(const struct device_node *np)
{
        return of_get_property(np, "device_type", NULL);
}

static inline bool of_node_is_type(const struct device_node *np, const char *type)
{
        const char *match = of_node_get_device_type(np);

        return np && match && type && !strcmp(match, type);
}

/**
 * of_parse_phandle - Resolve a phandle property to a device_node pointer
 * @np: Pointer to device node holding phandle property
 * @phandle_name: Name of property holding a phandle value
 * @index: For properties holding a table of phandles, this is the index into
 *         the table
 *
 * Return: The device_node pointer with refcount incremented.  Use
 * of_node_put() on it when done.
 */
static inline struct device_node *of_parse_phandle(const struct device_node *np,
                                                   const char *phandle_name,
                                                   int index)
{
        struct of_phandle_args args;

        if (__of_parse_phandle_with_args(np, phandle_name, NULL, 0,
                                         index, &args))
                return NULL;

        return args.np;
}

/**
 * of_parse_phandle_with_args() - Find a node pointed by phandle in a list
 * @np:                pointer to a device tree node containing a list
 * @list_name:        property name that contains a list
 * @cells_name:        property name that specifies phandles' arguments count
 * @index:        index of a phandle to parse out
 * @out_args:        optional pointer to output arguments structure (will be filled)
 *
 * This function is useful to parse lists of phandles and their arguments.
 * Returns 0 on success and fills out_args, on error returns appropriate
 * errno value.
 *
 * Caller is responsible to call of_node_put() on the returned out_args->np
 * pointer.
 *
 * Example::
 *
 *  phandle1: node1 {
 *        #list-cells = <2>;
 *  };
 *
 *  phandle2: node2 {
 *        #list-cells = <1>;
 *  };
 *
 *  node3 {
 *        list = <&phandle1 1 2 &phandle2 3>;
 *  };
 *
 * To get a device_node of the ``node2`` node you may call this:
 * of_parse_phandle_with_args(node3, "list", "#list-cells", 1, &args);
 */
static inline int of_parse_phandle_with_args(const struct device_node *np,
                                             const char *list_name,
                                             const char *cells_name,
                                             int index,
                                             struct of_phandle_args *out_args)
{
        int cell_count = -1;

        /* If cells_name is NULL we assume a cell count of 0 */
        if (!cells_name)
                cell_count = 0;

        return __of_parse_phandle_with_args(np, list_name, cells_name,
                                            cell_count, index, out_args);
}

/**
 * of_parse_phandle_with_fixed_args() - Find a node pointed by phandle in a list
 * @np:                pointer to a device tree node containing a list
 * @list_name:        property name that contains a list
 * @cell_count: number of argument cells following the phandle
 * @index:        index of a phandle to parse out
 * @out_args:        optional pointer to output arguments structure (will be filled)
 *
 * This function is useful to parse lists of phandles and their arguments.
 * Returns 0 on success and fills out_args, on error returns appropriate
 * errno value.
 *
 * Caller is responsible to call of_node_put() on the returned out_args->np
 * pointer.
 *
 * Example::
 *
 *  phandle1: node1 {
 *  };
 *
 *  phandle2: node2 {
 *  };
 *
 *  node3 {
 *        list = <&phandle1 0 2 &phandle2 2 3>;
 *  };
 *
 * To get a device_node of the ``node2`` node you may call this:
 * of_parse_phandle_with_fixed_args(node3, "list", 2, 1, &args);
 */
static inline int of_parse_phandle_with_fixed_args(const struct device_node *np,
                                                   const char *list_name,
                                                   int cell_count,
                                                   int index,
                                                   struct of_phandle_args *out_args)
{
        return __of_parse_phandle_with_args(np, list_name, NULL, cell_count,
                                            index, out_args);
}

/**
 * of_parse_phandle_with_optional_args() - Find a node pointed by phandle in a list
 * @np:                pointer to a device tree node containing a list
 * @list_name:        property name that contains a list
 * @cells_name:        property name that specifies phandles' arguments count
 * @index:        index of a phandle to parse out
 * @out_args:        optional pointer to output arguments structure (will be filled)
 *
 * Same as of_parse_phandle_with_args() except that if the cells_name property
 * is not found, cell_count of 0 is assumed.
 *
 * This is used to useful, if you have a phandle which didn't have arguments
 * before and thus doesn't have a '#*-cells' property but is now migrated to
 * having arguments while retaining backwards compatibility.
 */
static inline int of_parse_phandle_with_optional_args(const struct device_node *np,
                                                      const char *list_name,
                                                      const char *cells_name,
                                                      int index,
                                                      struct of_phandle_args *out_args)
{
        return __of_parse_phandle_with_args(np, list_name, cells_name,
                                            0, index, out_args);
}

/**
 * of_phandle_args_equal() - Compare two of_phandle_args
 * @a1:                First of_phandle_args to compare
 * @a2:                Second of_phandle_args to compare
 *
 * Return: True if a1 and a2 are the same (same node pointer, same phandle
 * args), false otherwise.
 */
static inline bool of_phandle_args_equal(const struct of_phandle_args *a1,
                                         const struct of_phandle_args *a2)
{
        return a1->np == a2->np &&
               a1->args_count == a2->args_count &&
               !memcmp(a1->args, a2->args, sizeof(a1->args[0]) * a1->args_count);
}

/**
 * of_property_count_u8_elems - Count the number of u8 elements in a property
 *
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 *
 * Search for a property in a device node and count the number of u8 elements
 * in it.
 *
 * Return: The number of elements on sucess, -EINVAL if the property does
 * not exist or its length does not match a multiple of u8 and -ENODATA if the
 * property does not have a value.
 */
static inline int of_property_count_u8_elems(const struct device_node *np,
                                const char *propname)
{
        return of_property_count_elems_of_size(np, propname, sizeof(u8));
}

/**
 * of_property_count_u16_elems - Count the number of u16 elements in a property
 *
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 *
 * Search for a property in a device node and count the number of u16 elements
 * in it.
 *
 * Return: The number of elements on sucess, -EINVAL if the property does
 * not exist or its length does not match a multiple of u16 and -ENODATA if the
 * property does not have a value.
 */
static inline int of_property_count_u16_elems(const struct device_node *np,
                                const char *propname)
{
        return of_property_count_elems_of_size(np, propname, sizeof(u16));
}

/**
 * of_property_count_u32_elems - Count the number of u32 elements in a property
 *
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 *
 * Search for a property in a device node and count the number of u32 elements
 * in it.
 *
 * Return: The number of elements on sucess, -EINVAL if the property does
 * not exist or its length does not match a multiple of u32 and -ENODATA if the
 * property does not have a value.
 */
static inline int of_property_count_u32_elems(const struct device_node *np,
                                const char *propname)
{
        return of_property_count_elems_of_size(np, propname, sizeof(u32));
}

/**
 * of_property_count_u64_elems - Count the number of u64 elements in a property
 *
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 *
 * Search for a property in a device node and count the number of u64 elements
 * in it.
 *
 * Return: The number of elements on sucess, -EINVAL if the property does
 * not exist or its length does not match a multiple of u64 and -ENODATA if the
 * property does not have a value.
 */
static inline int of_property_count_u64_elems(const struct device_node *np,
                                const char *propname)
{
        return of_property_count_elems_of_size(np, propname, sizeof(u64));
}

/**
 * of_property_read_string_array() - Read an array of strings from a multiple
 * strings property.
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 * @out_strs:        output array of string pointers.
 * @sz:                number of array elements to read.
 *
 * Search for a property in a device tree node and retrieve a list of
 * terminated string values (pointer to data, not a copy) in that property.
 *
 * Return: If @out_strs is NULL, the number of strings in the property is returned.
 */
static inline int of_property_read_string_array(const struct device_node *np,
                                                const char *propname, const char **out_strs,
                                                size_t sz)
{
        return of_property_read_string_helper(np, propname, out_strs, sz, 0);
}

/**
 * of_property_count_strings() - Find and return the number of strings from a
 * multiple strings property.
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 *
 * Search for a property in a device tree node and retrieve the number of null
 * terminated string contain in it.
 *
 * Return: The number of strings on success, -EINVAL if the property does not
 * exist, -ENODATA if property does not have a value, and -EILSEQ if the string
 * is not null-terminated within the length of the property data.
 */
static inline int of_property_count_strings(const struct device_node *np,
                                            const char *propname)
{
        return of_property_read_string_helper(np, propname, NULL, 0, 0);
}

/**
 * of_property_read_string_index() - Find and read a string from a multiple
 * strings property.
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 * @index:        index of the string in the list of strings
 * @output:        pointer to null terminated return string, modified only if
 *                return value is 0.
 *
 * Search for a property in a device tree node and retrieve a null
 * terminated string value (pointer to data, not a copy) in the list of strings
 * contained in that property.
 *
 * Return: 0 on success, -EINVAL if the property does not exist, -ENODATA if
 * property does not have a value, and -EILSEQ if the string is not
 * null-terminated within the length of the property data.
 *
 * The out_string pointer is modified only if a valid string can be decoded.
 */
static inline int of_property_read_string_index(const struct device_node *np,
                                                const char *propname,
                                                int index, const char **output)
{
        int rc = of_property_read_string_helper(np, propname, output, 1, index);
        return rc < 0 ? rc : 0;
}

/**
 * of_property_present - Test if a property is present in a node
 * @np:                device node to search for the property.
 * @propname:        name of the property to be searched.
 *
 * Test for a property present in a device node.
 *
 * Return: true if the property exists false otherwise.
 */
static inline bool of_property_present(const struct device_node *np, const char *propname)
{
        struct property *prop = of_find_property(np, propname, NULL);

        return prop ? true : false;
}

/**
 * of_property_read_u8_array - Find and read an array of u8 from a property.
 *
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 * @out_values:        pointer to return value, modified only if return value is 0.
 * @sz:                number of array elements to read
 *
 * Search for a property in a device node and read 8-bit value(s) from
 * it.
 *
 * dts entry of array should be like:
 *  ``property = /bits/ 8 <0x50 0x60 0x70>;``
 *
 * Return: 0 on success, -EINVAL if the property does not exist,
 * -ENODATA if property does not have a value, and -EOVERFLOW if the
 * property data isn't large enough.
 *
 * The out_values is modified only if a valid u8 value can be decoded.
 */
static inline int of_property_read_u8_array(const struct device_node *np,
                                            const char *propname,
                                            u8 *out_values, size_t sz)
{
        int ret = of_property_read_variable_u8_array(np, propname, out_values,
                                                     sz, 0);
        if (ret >= 0)
                return 0;
        else
                return ret;
}

/**
 * of_property_read_u16_array - Find and read an array of u16 from a property.
 *
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 * @out_values:        pointer to return value, modified only if return value is 0.
 * @sz:                number of array elements to read
 *
 * Search for a property in a device node and read 16-bit value(s) from
 * it.
 *
 * dts entry of array should be like:
 *  ``property = /bits/ 16 <0x5000 0x6000 0x7000>;``
 *
 * Return: 0 on success, -EINVAL if the property does not exist,
 * -ENODATA if property does not have a value, and -EOVERFLOW if the
 * property data isn't large enough.
 *
 * The out_values is modified only if a valid u16 value can be decoded.
 */
static inline int of_property_read_u16_array(const struct device_node *np,
                                             const char *propname,
                                             u16 *out_values, size_t sz)
{
        int ret = of_property_read_variable_u16_array(np, propname, out_values,
                                                      sz, 0);
        if (ret >= 0)
                return 0;
        else
                return ret;
}

/**
 * of_property_read_u32_array - Find and read an array of 32 bit integers
 * from a property.
 *
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 * @out_values:        pointer to return value, modified only if return value is 0.
 * @sz:                number of array elements to read
 *
 * Search for a property in a device node and read 32-bit value(s) from
 * it.
 *
 * Return: 0 on success, -EINVAL if the property does not exist,
 * -ENODATA if property does not have a value, and -EOVERFLOW if the
 * property data isn't large enough.
 *
 * The out_values is modified only if a valid u32 value can be decoded.
 */
static inline int of_property_read_u32_array(const struct device_node *np,
                                             const char *propname,
                                             u32 *out_values, size_t sz)
{
        int ret = of_property_read_variable_u32_array(np, propname, out_values,
                                                      sz, 0);
        if (ret >= 0)
                return 0;
        else
                return ret;
}

/**
 * of_property_read_u64_array - Find and read an array of 64 bit integers
 * from a property.
 *
 * @np:                device node from which the property value is to be read.
 * @propname:        name of the property to be searched.
 * @out_values:        pointer to return value, modified only if return value is 0.
 * @sz:                number of array elements to read
 *
 * Search for a property in a device node and read 64-bit value(s) from
 * it.
 *
 * Return: 0 on success, -EINVAL if the property does not exist,
 * -ENODATA if property does not have a value, and -EOVERFLOW if the
 * property data isn't large enough.
 *
 * The out_values is modified only if a valid u64 value can be decoded.
 */
static inline int of_property_read_u64_array(const struct device_node *np,
                                             const char *propname,
                                             u64 *out_values, size_t sz)
{
        int ret = of_property_read_variable_u64_array(np, propname, out_values,
                                                      sz, 0);
        if (ret >= 0)
                return 0;
        else
                return ret;
}

static inline int of_property_read_u8(const struct device_node *np,
                                       const char *propname,
                                       u8 *out_value)
{
        return of_property_read_u8_array(np, propname, out_value, 1);
}

static inline int of_property_read_u16(const struct device_node *np,
                                       const char *propname,
                                       u16 *out_value)
{
        return of_property_read_u16_array(np, propname, out_value, 1);
}

static inline int of_property_read_u32(const struct device_node *np,
                                       const char *propname,
                                       u32 *out_value)
{
        return of_property_read_u32_array(np, propname, out_value, 1);
}

static inline int of_property_read_s32(const struct device_node *np,
                                       const char *propname,
                                       s32 *out_value)
{
        return of_property_read_u32(np, propname, (u32*) out_value);
}

#define of_for_each_phandle(it, err, np, ln, cn, cc)                        \
        for (of_phandle_iterator_init((it), (np), (ln), (cn), (cc)),        \
             err = of_phandle_iterator_next(it);                        \
             err == 0;                                                        \
             err = of_phandle_iterator_next(it))

#define of_property_for_each_u32(np, propname, u)                        \
        for (struct {const struct property *prop; const __be32 *item; } _it =        \
                {of_find_property(np, propname, NULL),                        \
                 of_prop_next_u32(_it.prop, NULL, &u)};                        \
             _it.item;                                                        \
             _it.item = of_prop_next_u32(_it.prop, _it.item, &u))

#define of_property_for_each_string(np, propname, prop, s)        \
        for (prop = of_find_property(np, propname, NULL),        \
                s = of_prop_next_string(prop, NULL);                \
                s;                                                \
                s = of_prop_next_string(prop, s))

#define for_each_node_by_name(dn, name) \
        for (dn = of_find_node_by_name(NULL, name); dn; \
             dn = of_find_node_by_name(dn, name))
#define for_each_node_by_type(dn, type) \
        for (dn = of_find_node_by_type(NULL, type); dn; \
             dn = of_find_node_by_type(dn, type))
#define for_each_compatible_node(dn, type, compatible) \
        for (dn = of_find_compatible_node(NULL, type, compatible); dn; \
             dn = of_find_compatible_node(dn, type, compatible))
#define for_each_matching_node(dn, matches) \
        for (dn = of_find_matching_node(NULL, matches); dn; \
             dn = of_find_matching_node(dn, matches))
#define for_each_matching_node_and_match(dn, matches, match) \
        for (dn = of_find_matching_node_and_match(NULL, matches, match); \
             dn; dn = of_find_matching_node_and_match(dn, matches, match))

#define for_each_child_of_node(parent, child) \
        for (child = of_get_next_child(parent, NULL); child != NULL; \
             child = of_get_next_child(parent, child))

#define for_each_child_of_node_scoped(parent, child) \
        for (struct device_node *child __free(device_node) =                \
             of_get_next_child(parent, NULL);                                \
             child != NULL;                                                \
             child = of_get_next_child(parent, child))

#define for_each_child_of_node_with_prefix(parent, child, prefix)        \
        for (struct device_node *child __free(device_node) =                \
             of_get_next_child_with_prefix(parent, NULL, prefix);        \
             child != NULL;                                                \
             child = of_get_next_child_with_prefix(parent, child, prefix))

#define for_each_available_child_of_node(parent, child) \
        for (child = of_get_next_available_child(parent, NULL); child != NULL; \
             child = of_get_next_available_child(parent, child))
#define for_each_reserved_child_of_node(parent, child)                        \
        for (child = of_get_next_reserved_child(parent, NULL); child != NULL; \
             child = of_get_next_reserved_child(parent, child))

#define for_each_available_child_of_node_scoped(parent, child) \
        for (struct device_node *child __free(device_node) =                \
             of_get_next_available_child(parent, NULL);                        \
             child != NULL;                                                \
             child = of_get_next_available_child(parent, child))

#define for_each_of_cpu_node(cpu) \
        for (cpu = of_get_next_cpu_node(NULL); cpu != NULL; \
             cpu = of_get_next_cpu_node(cpu))

#define for_each_node_with_property(dn, prop_name) \
        for (dn = of_find_node_with_property(NULL, prop_name); dn; \
             dn = of_find_node_with_property(dn, prop_name))

static inline int of_get_child_count(const struct device_node *np)
{
        struct device_node *child;
        int num = 0;

        for_each_child_of_node(np, child)
                num++;

        return num;
}

static inline int of_get_available_child_count(const struct device_node *np)
{
        struct device_node *child;
        int num = 0;

        for_each_available_child_of_node(np, child)
                num++;

        return num;
}

#define _OF_DECLARE_STUB(table, name, compat, fn, fn_type)                \
        static const struct of_device_id __of_table_##name                \
                __attribute__((unused))                                        \
                 = { .compatible = compat,                                \
                     .data = (fn == (fn_type)NULL) ? fn : fn }

#if defined(CONFIG_OF) && !defined(MODULE)
#define _OF_DECLARE(table, name, compat, fn, fn_type)                        \
        static const struct of_device_id __of_table_##name                \
                __used __section("__" #table "_of_table")                \
                __aligned(__alignof__(struct of_device_id))                \
                 = { .compatible = compat,                                \
                     .data = (fn == (fn_type)NULL) ? fn : fn  }
#else
#define _OF_DECLARE(table, name, compat, fn, fn_type)                        \
        _OF_DECLARE_STUB(table, name, compat, fn, fn_type)
#endif

typedef int (*of_init_fn_2)(struct device_node *, struct device_node *);
typedef int (*of_init_fn_1_ret)(struct device_node *);
typedef void (*of_init_fn_1)(struct device_node *);

#define OF_DECLARE_1(table, name, compat, fn) \
                _OF_DECLARE(table, name, compat, fn, of_init_fn_1)
#define OF_DECLARE_1_RET(table, name, compat, fn) \
                _OF_DECLARE(table, name, compat, fn, of_init_fn_1_ret)
#define OF_DECLARE_2(table, name, compat, fn) \
                _OF_DECLARE(table, name, compat, fn, of_init_fn_2)

/**
 * struct of_changeset_entry        - Holds a changeset entry
 *
 * @node:        list_head for the log list
 * @action:        notifier action
 * @np:                pointer to the device node affected
 * @prop:        pointer to the property affected
 * @old_prop:        hold a pointer to the original property
 *
 * Every modification of the device tree during a changeset
 * is held in a list of of_changeset_entry structures.
 * That way we can recover from a partial application, or we can
 * revert the changeset
 */
struct of_changeset_entry {
        struct list_head node;
        unsigned long action;
        struct device_node *np;
        struct property *prop;
        struct property *old_prop;
};

/**
 * struct of_changeset - changeset tracker structure
 *
 * @entries:        list_head for the changeset entries
 *
 * changesets are a convenient way to apply bulk changes to the
 * live tree. In case of an error, changes are rolled-back.
 * changesets live on after initial application, and if not
 * destroyed after use, they can be reverted in one single call.
 */
struct of_changeset {
        struct list_head entries;
};

enum of_reconfig_change {
        OF_RECONFIG_NO_CHANGE = 0,
        OF_RECONFIG_CHANGE_ADD,
        OF_RECONFIG_CHANGE_REMOVE,
};

struct notifier_block;

#ifdef CONFIG_OF_DYNAMIC
extern int of_reconfig_notifier_register(struct notifier_block *);
extern int of_reconfig_notifier_unregister(struct notifier_block *);
extern int of_reconfig_notify(unsigned long, struct of_reconfig_data *rd);
extern int of_reconfig_get_state_change(unsigned long action,
                                        struct of_reconfig_data *arg);

extern void of_changeset_init(struct of_changeset *ocs);
extern void of_changeset_destroy(struct of_changeset *ocs);
extern int of_changeset_apply(struct of_changeset *ocs);
extern int of_changeset_revert(struct of_changeset *ocs);
extern int of_changeset_action(struct of_changeset *ocs,
                unsigned long action, struct device_node *np,
                struct property *prop);

static inline int of_changeset_attach_node(struct of_changeset *ocs,
                struct device_node *np)
{
        return of_changeset_action(ocs, OF_RECONFIG_ATTACH_NODE, np, NULL);
}

static inline int of_changeset_detach_node(struct of_changeset *ocs,
                struct device_node *np)
{
        return of_changeset_action(ocs, OF_RECONFIG_DETACH_NODE, np, NULL);
}

static inline int of_changeset_add_property(struct of_changeset *ocs,
                struct device_node *np, struct property *prop)
{
        return of_changeset_action(ocs, OF_RECONFIG_ADD_PROPERTY, np, prop);
}

static inline int of_changeset_remove_property(struct of_changeset *ocs,
                struct device_node *np, struct property *prop)
{
        return of_changeset_action(ocs, OF_RECONFIG_REMOVE_PROPERTY, np, prop);
}

static inline int of_changeset_update_property(struct of_changeset *ocs,
                struct device_node *np, struct property *prop)
{
        return of_changeset_action(ocs, OF_RECONFIG_UPDATE_PROPERTY, np, prop);
}

struct device_node *of_changeset_create_node(struct of_changeset *ocs,
                                             struct device_node *parent,
                                             const char *full_name);
int of_changeset_add_prop_string(struct of_changeset *ocs,
                                 struct device_node *np,
                                 const char *prop_name, const char *str);
int of_changeset_add_prop_string_array(struct of_changeset *ocs,
                                       struct device_node *np,
                                       const char *prop_name,
                                       const char * const *str_array, size_t sz);
int of_changeset_add_prop_u32_array(struct of_changeset *ocs,
                                    struct device_node *np,
                                    const char *prop_name,
                                    const u32 *array, size_t sz);
static inline int of_changeset_add_prop_u32(struct of_changeset *ocs,
                                            struct device_node *np,
                                            const char *prop_name,
                                            const u32 val)
{
        return of_changeset_add_prop_u32_array(ocs, np, prop_name, &val, 1);
}

int of_changeset_update_prop_string(struct of_changeset *ocs,
                                    struct device_node *np,
                                    const char *prop_name, const char *str);

int of_changeset_add_prop_bool(struct of_changeset *ocs, struct device_node *np,
                               const char *prop_name);

#else /* CONFIG_OF_DYNAMIC */
static inline int of_reconfig_notifier_register(struct notifier_block *nb)
{
        return -EINVAL;
}
static inline int of_reconfig_notifier_unregister(struct notifier_block *nb)
{
        return -EINVAL;
}
static inline int of_reconfig_notify(unsigned long action,
                                     struct of_reconfig_data *arg)
{
        return -EINVAL;
}
static inline int of_reconfig_get_state_change(unsigned long action,
                                                struct of_reconfig_data *arg)
{
        return -EINVAL;
}
#endif /* CONFIG_OF_DYNAMIC */

/**
 * of_device_is_system_power_controller - Tells if system-power-controller is found for device_node
 * @np: Pointer to the given device_node
 *
 * Return: true if present false otherwise
 */
static inline bool of_device_is_system_power_controller(const struct device_node *np)
{
        return of_property_read_bool(np, "system-power-controller");
}

/**
 * of_have_populated_dt() - Has DT been populated by bootloader
 *
 * Return: True if a DTB has been populated by the bootloader and it isn't the
 * empty builtin one. False otherwise.
 */
static inline bool of_have_populated_dt(void)
{
#ifdef CONFIG_OF
        return of_property_present(of_root, "compatible");
#else
        return false;
#endif
}

/*
 * Overlay support
 */

enum of_overlay_notify_action {
        OF_OVERLAY_INIT = 0,        /* kzalloc() of ovcs sets this value */
        OF_OVERLAY_PRE_APPLY,
        OF_OVERLAY_POST_APPLY,
        OF_OVERLAY_PRE_REMOVE,
        OF_OVERLAY_POST_REMOVE,
};

static inline const char *of_overlay_action_name(enum of_overlay_notify_action action)
{
        static const char *const of_overlay_action_name[] = {
                "init",
                "pre-apply",
                "post-apply",
                "pre-remove",
                "post-remove",
        };

        return of_overlay_action_name[action];
}

struct of_overlay_notify_data {
        struct device_node *overlay;
        struct device_node *target;
};

#ifdef CONFIG_OF_OVERLAY

int of_overlay_fdt_apply(const void *overlay_fdt, u32 overlay_fdt_size,
                         int *ovcs_id, const struct device_node *target_base);
int of_overlay_remove(int *ovcs_id);
int of_overlay_remove_all(void);

int of_overlay_notifier_register(struct notifier_block *nb);
int of_overlay_notifier_unregister(struct notifier_block *nb);

#else

static inline int of_overlay_fdt_apply(const void *overlay_fdt, u32 overlay_fdt_size,
                                       int *ovcs_id, const struct device_node *target_base)
{
        return -ENOTSUPP;
}

static inline int of_overlay_remove(int *ovcs_id)
{
        return -ENOTSUPP;
}

static inline int of_overlay_remove_all(void)
{
        return -ENOTSUPP;
}

static inline int of_overlay_notifier_register(struct notifier_block *nb)
{
        return 0;
}

static inline int of_overlay_notifier_unregister(struct notifier_block *nb)
{
        return 0;
}

#endif

#endif /* _LINUX_OF_H */















   43 









   34 

    9 









  203 















  203 











   92 



   92 
   15 
   11 







   13 



   13 

    1 
    1 







    9 



    9 
















    7 


   11 
    4 


    4 






   11 






   11 

    2 










   15 



   15 

   15 


   15 



   15 











   15 








   15 


    2 





    6 



    6 













   82 









    8 



   80 
   81 
   81 



   80 

   81 






   81 
   81 
   39 

   39 
   39 









   34 




   80 




   81 









   81 










    4 



    2 

    2 












   22 








   22 



   14 
    4 




   14 






   22 





    2 






















   21 








   13 
    3 
    5 



   16 
   16 
   10 





   16 



   16 




























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
/* Linux multicast routing support
 * Common logic shared by IPv4 [ipmr] and IPv6 [ip6mr] implementation
 */

#include <linux/rhashtable.h>
#include <linux/mroute_base.h>

/* Sets everything common except 'dev', since that is done under locking */
void vif_device_init(struct vif_device *v,
                     struct net_device *dev,
                     unsigned long rate_limit,
                     unsigned char threshold,
                     unsigned short flags,
                     unsigned short get_iflink_mask)
{
        RCU_INIT_POINTER(v->dev, NULL);
        v->bytes_in = 0;
        v->bytes_out = 0;
        v->pkt_in = 0;
        v->pkt_out = 0;
        v->rate_limit = rate_limit;
        v->flags = flags;
        v->threshold = threshold;
        if (v->flags & get_iflink_mask)
                v->link = dev_get_iflink(dev);
        else
                v->link = dev->ifindex;
}
EXPORT_SYMBOL(vif_device_init);

struct mr_table *
mr_table_alloc(struct net *net, u32 id,
               struct mr_table_ops *ops,
               void (*expire_func)(struct timer_list *t),
               void (*table_set)(struct mr_table *mrt,
                                 struct net *net))
{
        struct mr_table *mrt;
        int err;

        mrt = kzalloc(sizeof(*mrt), GFP_KERNEL);
        if (!mrt)
                return ERR_PTR(-ENOMEM);
        mrt->id = id;
        write_pnet(&mrt->net, net);

        mrt->ops = *ops;
        err = rhltable_init(&mrt->mfc_hash, mrt->ops.rht_params);
        if (err) {
                kfree(mrt);
                return ERR_PTR(err);
        }
        INIT_LIST_HEAD(&mrt->mfc_cache_list);
        INIT_LIST_HEAD(&mrt->mfc_unres_queue);

        timer_setup(&mrt->ipmr_expire_timer, expire_func, 0);

        mrt->mroute_reg_vif_num = -1;
        table_set(mrt, net);
        return mrt;
}
EXPORT_SYMBOL(mr_table_alloc);

void *mr_mfc_find_parent(struct mr_table *mrt, void *hasharg, int parent)
{
        struct rhlist_head *tmp, *list;
        struct mr_mfc *c;

        list = rhltable_lookup(&mrt->mfc_hash, hasharg, *mrt->ops.rht_params);
        rhl_for_each_entry_rcu(c, tmp, list, mnode)
                if (parent == -1 || parent == c->mfc_parent)
                        return c;

        return NULL;
}
EXPORT_SYMBOL(mr_mfc_find_parent);

void *mr_mfc_find_any_parent(struct mr_table *mrt, int vifi)
{
        struct rhlist_head *tmp, *list;
        struct mr_mfc *c;

        list = rhltable_lookup(&mrt->mfc_hash, mrt->ops.cmparg_any,
                               *mrt->ops.rht_params);
        rhl_for_each_entry_rcu(c, tmp, list, mnode)
                if (c->mfc_un.res.ttls[vifi] < 255)
                        return c;

        return NULL;
}
EXPORT_SYMBOL(mr_mfc_find_any_parent);

void *mr_mfc_find_any(struct mr_table *mrt, int vifi, void *hasharg)
{
        struct rhlist_head *tmp, *list;
        struct mr_mfc *c, *proxy;

        list = rhltable_lookup(&mrt->mfc_hash, hasharg, *mrt->ops.rht_params);
        rhl_for_each_entry_rcu(c, tmp, list, mnode) {
                if (c->mfc_un.res.ttls[vifi] < 255)
                        return c;

                /* It's ok if the vifi is part of the static tree */
                proxy = mr_mfc_find_any_parent(mrt, c->mfc_parent);
                if (proxy && proxy->mfc_un.res.ttls[vifi] < 255)
                        return c;
        }

        return mr_mfc_find_any_parent(mrt, vifi);
}
EXPORT_SYMBOL(mr_mfc_find_any);

#ifdef CONFIG_PROC_FS
void *mr_vif_seq_idx(struct net *net, struct mr_vif_iter *iter, loff_t pos)
{
        struct mr_table *mrt = iter->mrt;

        for (iter->ct = 0; iter->ct < mrt->maxvif; ++iter->ct) {
                if (!VIF_EXISTS(mrt, iter->ct))
                        continue;
                if (pos-- == 0)
                        return &mrt->vif_table[iter->ct];
        }
        return NULL;
}
EXPORT_SYMBOL(mr_vif_seq_idx);

void *mr_vif_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct mr_vif_iter *iter = seq->private;
        struct net *net = seq_file_net(seq);
        struct mr_table *mrt = iter->mrt;

        ++*pos;
        if (v == SEQ_START_TOKEN)
                return mr_vif_seq_idx(net, iter, 0);

        while (++iter->ct < mrt->maxvif) {
                if (!VIF_EXISTS(mrt, iter->ct))
                        continue;
                return &mrt->vif_table[iter->ct];
        }
        return NULL;
}
EXPORT_SYMBOL(mr_vif_seq_next);

void *mr_mfc_seq_idx(struct net *net,
                     struct mr_mfc_iter *it, loff_t pos)
{
        struct mr_table *mrt = it->mrt;
        struct mr_mfc *mfc;

        rcu_read_lock();
        it->cache = &mrt->mfc_cache_list;
        list_for_each_entry_rcu(mfc, &mrt->mfc_cache_list, list)
                if (pos-- == 0)
                        return mfc;
        rcu_read_unlock();

        spin_lock_bh(it->lock);
        it->cache = &mrt->mfc_unres_queue;
        list_for_each_entry(mfc, it->cache, list)
                if (pos-- == 0)
                        return mfc;
        spin_unlock_bh(it->lock);

        it->cache = NULL;
        return NULL;
}
EXPORT_SYMBOL(mr_mfc_seq_idx);

void *mr_mfc_seq_next(struct seq_file *seq, void *v,
                      loff_t *pos)
{
        struct mr_mfc_iter *it = seq->private;
        struct net *net = seq_file_net(seq);
        struct mr_table *mrt = it->mrt;
        struct mr_mfc *c = v;

        ++*pos;

        if (v == SEQ_START_TOKEN)
                return mr_mfc_seq_idx(net, seq->private, 0);

        if (c->list.next != it->cache)
                return list_entry(c->list.next, struct mr_mfc, list);

        if (it->cache == &mrt->mfc_unres_queue)
                goto end_of_list;

        /* exhausted cache_array, show unresolved */
        rcu_read_unlock();
        it->cache = &mrt->mfc_unres_queue;

        spin_lock_bh(it->lock);
        if (!list_empty(it->cache))
                return list_first_entry(it->cache, struct mr_mfc, list);

end_of_list:
        spin_unlock_bh(it->lock);
        it->cache = NULL;

        return NULL;
}
EXPORT_SYMBOL(mr_mfc_seq_next);
#endif

int mr_fill_mroute(struct mr_table *mrt, struct sk_buff *skb,
                   struct mr_mfc *c, struct rtmsg *rtm)
{
        struct net_device *vif_dev;
        struct rta_mfc_stats mfcs;
        struct nlattr *mp_attr;
        struct rtnexthop *nhp;
        unsigned long lastuse;
        int ct;

        /* If cache is unresolved, don't try to parse IIF and OIF */
        if (c->mfc_parent >= MAXVIFS) {
                rtm->rtm_flags |= RTNH_F_UNRESOLVED;
                return -ENOENT;
        }

        rcu_read_lock();
        vif_dev = rcu_dereference(mrt->vif_table[c->mfc_parent].dev);
        if (vif_dev && nla_put_u32(skb, RTA_IIF, vif_dev->ifindex) < 0) {
                rcu_read_unlock();
                return -EMSGSIZE;
        }
        rcu_read_unlock();

        if (c->mfc_flags & MFC_OFFLOAD)
                rtm->rtm_flags |= RTNH_F_OFFLOAD;

        mp_attr = nla_nest_start_noflag(skb, RTA_MULTIPATH);
        if (!mp_attr)
                return -EMSGSIZE;

        rcu_read_lock();
        for (ct = c->mfc_un.res.minvif; ct < c->mfc_un.res.maxvif; ct++) {
                struct vif_device *vif = &mrt->vif_table[ct];

                vif_dev = rcu_dereference(vif->dev);
                if (vif_dev && c->mfc_un.res.ttls[ct] < 255) {

                        nhp = nla_reserve_nohdr(skb, sizeof(*nhp));
                        if (!nhp) {
                                rcu_read_unlock();
                                nla_nest_cancel(skb, mp_attr);
                                return -EMSGSIZE;
                        }

                        nhp->rtnh_flags = 0;
                        nhp->rtnh_hops = c->mfc_un.res.ttls[ct];
                        nhp->rtnh_ifindex = vif_dev->ifindex;
                        nhp->rtnh_len = sizeof(*nhp);
                }
        }
        rcu_read_unlock();

        nla_nest_end(skb, mp_attr);

        lastuse = READ_ONCE(c->mfc_un.res.lastuse);
        lastuse = time_after_eq(jiffies, lastuse) ? jiffies - lastuse : 0;

        mfcs.mfcs_packets = atomic_long_read(&c->mfc_un.res.pkt);
        mfcs.mfcs_bytes = atomic_long_read(&c->mfc_un.res.bytes);
        mfcs.mfcs_wrong_if = atomic_long_read(&c->mfc_un.res.wrong_if);
        if (nla_put_64bit(skb, RTA_MFC_STATS, sizeof(mfcs), &mfcs, RTA_PAD) ||
            nla_put_u64_64bit(skb, RTA_EXPIRES, jiffies_to_clock_t(lastuse),
                              RTA_PAD))
                return -EMSGSIZE;

        rtm->rtm_type = RTN_MULTICAST;
        return 1;
}
EXPORT_SYMBOL(mr_fill_mroute);

static bool mr_mfc_uses_dev(const struct mr_table *mrt,
                            const struct mr_mfc *c,
                            const struct net_device *dev)
{
        int ct;

        for (ct = c->mfc_un.res.minvif; ct < c->mfc_un.res.maxvif; ct++) {
                const struct net_device *vif_dev;
                const struct vif_device *vif;

                vif = &mrt->vif_table[ct];
                vif_dev = rcu_access_pointer(vif->dev);
                if (vif_dev && c->mfc_un.res.ttls[ct] < 255 &&
                    vif_dev == dev)
                        return true;
        }
        return false;
}

int mr_table_dump(struct mr_table *mrt, struct sk_buff *skb,
                  struct netlink_callback *cb,
                  int (*fill)(struct mr_table *mrt, struct sk_buff *skb,
                              u32 portid, u32 seq, struct mr_mfc *c,
                              int cmd, int flags),
                  spinlock_t *lock, struct fib_dump_filter *filter)
{
        unsigned int e = 0, s_e = cb->args[1];
        unsigned int flags = NLM_F_MULTI;
        struct mr_mfc *mfc;
        int err;

        if (filter->filter_set)
                flags |= NLM_F_DUMP_FILTERED;

        list_for_each_entry_rcu(mfc, &mrt->mfc_cache_list, list,
                                lockdep_rtnl_is_held()) {
                if (e < s_e)
                        goto next_entry;
                if (filter->dev &&
                    !mr_mfc_uses_dev(mrt, mfc, filter->dev))
                        goto next_entry;

                err = fill(mrt, skb, NETLINK_CB(cb->skb).portid,
                           cb->nlh->nlmsg_seq, mfc, RTM_NEWROUTE, flags);
                if (err < 0)
                        goto out;
next_entry:
                e++;
        }

        spin_lock_bh(lock);
        list_for_each_entry(mfc, &mrt->mfc_unres_queue, list) {
                if (e < s_e)
                        goto next_entry2;

                err = fill(mrt, skb, NETLINK_CB(cb->skb).portid,
                           cb->nlh->nlmsg_seq, mfc, RTM_NEWROUTE, flags);
                if (err < 0) {
                        spin_unlock_bh(lock);
                        goto out;
                }
next_entry2:
                e++;
        }
        spin_unlock_bh(lock);
        err = 0;
out:
        cb->args[1] = e;
        return err;
}
EXPORT_SYMBOL(mr_table_dump);

int mr_rtm_dumproute(struct sk_buff *skb, struct netlink_callback *cb,
                     struct mr_table *(*iter)(struct net *net,
                                              struct mr_table *mrt),
                     int (*fill)(struct mr_table *mrt,
                                 struct sk_buff *skb,
                                 u32 portid, u32 seq, struct mr_mfc *c,
                                 int cmd, int flags),
                     spinlock_t *lock, struct fib_dump_filter *filter)
{
        unsigned int t = 0, s_t = cb->args[0];
        struct net *net = sock_net(skb->sk);
        struct mr_table *mrt;
        int err;

        /* multicast does not track protocol or have route type other
         * than RTN_MULTICAST
         */
        if (filter->filter_set) {
                if (filter->protocol || filter->flags ||
                    (filter->rt_type && filter->rt_type != RTN_MULTICAST))
                        return skb->len;
        }

        rcu_read_lock();
        for (mrt = iter(net, NULL); mrt; mrt = iter(net, mrt)) {
                if (t < s_t)
                        goto next_table;

                err = mr_table_dump(mrt, skb, cb, fill, lock, filter);
                if (err < 0)
                        break;
                cb->args[1] = 0;
next_table:
                t++;
        }
        rcu_read_unlock();

        cb->args[0] = t;

        return skb->len;
}
EXPORT_SYMBOL(mr_rtm_dumproute);

int mr_dump(struct net *net, struct notifier_block *nb, unsigned short family,
            int (*rules_dump)(struct net *net,
                              struct notifier_block *nb,
                              struct netlink_ext_ack *extack),
            struct mr_table *(*mr_iter)(struct net *net,
                                        struct mr_table *mrt),
            struct netlink_ext_ack *extack)
{
        struct mr_table *mrt;
        int err;

        err = rules_dump(net, nb, extack);
        if (err)
                return err;

        for (mrt = mr_iter(net, NULL); mrt; mrt = mr_iter(net, mrt)) {
                struct vif_device *v = &mrt->vif_table[0];
                struct net_device *vif_dev;
                struct mr_mfc *mfc;
                int vifi;

                /* Notifiy on table VIF entries */
                rcu_read_lock();
                for (vifi = 0; vifi < mrt->maxvif; vifi++, v++) {
                        vif_dev = rcu_dereference(v->dev);
                        if (!vif_dev)
                                continue;

                        err = mr_call_vif_notifier(nb, family,
                                                   FIB_EVENT_VIF_ADD, v,
                                                   vif_dev, vifi,
                                                   mrt->id, extack);
                        if (err)
                                break;
                }
                rcu_read_unlock();

                if (err)
                        return err;

                /* Notify on table MFC entries */
                list_for_each_entry_rcu(mfc, &mrt->mfc_cache_list, list) {
                        err = mr_call_mfc_notifier(nb, family,
                                                   FIB_EVENT_ENTRY_ADD,
                                                   mfc, mrt->id, extack);
                        if (err)
                                return err;
                }
        }

        return 0;
}
EXPORT_SYMBOL(mr_dump);























   27 



   27 
   27 













    4 
    4 






















































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2018 Facebook
 */
#include <linux/bpf.h>
#include <linux/err.h>
#include <linux/sock_diag.h>
#include <net/sock_reuseport.h>
#include <linux/btf_ids.h>

struct reuseport_array {
        struct bpf_map map;
        struct sock __rcu *ptrs[];
};

static struct reuseport_array *reuseport_array(struct bpf_map *map)
{
        return (struct reuseport_array *)map;
}

/* The caller must hold the reuseport_lock */
void bpf_sk_reuseport_detach(struct sock *sk)
{
        struct sock __rcu **socks;

        write_lock_bh(&sk->sk_callback_lock);
        socks = __locked_read_sk_user_data_with_flags(sk, SK_USER_DATA_BPF);
        if (socks) {
                WRITE_ONCE(sk->sk_user_data, NULL);
                /*
                 * Do not move this NULL assignment outside of
                 * sk->sk_callback_lock because there is
                 * a race with reuseport_array_free()
                 * which does not hold the reuseport_lock.
                 */
                RCU_INIT_POINTER(*socks, NULL);
        }
        write_unlock_bh(&sk->sk_callback_lock);
}

static int reuseport_array_alloc_check(union bpf_attr *attr)
{
        if (attr->value_size != sizeof(u32) &&
            attr->value_size != sizeof(u64))
                return -EINVAL;

        return array_map_alloc_check(attr);
}

static void *reuseport_array_lookup_elem(struct bpf_map *map, void *key)
{
        struct reuseport_array *array = reuseport_array(map);
        u32 index = *(u32 *)key;

        if (unlikely(index >= array->map.max_entries))
                return NULL;

        return rcu_dereference(array->ptrs[index]);
}

/* Called from syscall only */
static long reuseport_array_delete_elem(struct bpf_map *map, void *key)
{
        struct reuseport_array *array = reuseport_array(map);
        u32 index = *(u32 *)key;
        struct sock *sk;
        int err;

        if (index >= map->max_entries)
                return -E2BIG;

        if (!rcu_access_pointer(array->ptrs[index]))
                return -ENOENT;

        spin_lock_bh(&reuseport_lock);

        sk = rcu_dereference_protected(array->ptrs[index],
                                       lockdep_is_held(&reuseport_lock));
        if (sk) {
                write_lock_bh(&sk->sk_callback_lock);
                WRITE_ONCE(sk->sk_user_data, NULL);
                RCU_INIT_POINTER(array->ptrs[index], NULL);
                write_unlock_bh(&sk->sk_callback_lock);
                err = 0;
        } else {
                err = -ENOENT;
        }

        spin_unlock_bh(&reuseport_lock);

        return err;
}

static void reuseport_array_free(struct bpf_map *map)
{
        struct reuseport_array *array = reuseport_array(map);
        struct sock *sk;
        u32 i;

        /*
         * ops->map_*_elem() will not be able to access this
         * array now. Hence, this function only races with
         * bpf_sk_reuseport_detach() which was triggered by
         * close() or disconnect().
         *
         * This function and bpf_sk_reuseport_detach() are
         * both removing sk from "array".  Who removes it
         * first does not matter.
         *
         * The only concern here is bpf_sk_reuseport_detach()
         * may access "array" which is being freed here.
         * bpf_sk_reuseport_detach() access this "array"
         * through sk->sk_user_data _and_ with sk->sk_callback_lock
         * held which is enough because this "array" is not freed
         * until all sk->sk_user_data has stopped referencing this "array".
         *
         * Hence, due to the above, taking "reuseport_lock" is not
         * needed here.
         */

        /*
         * Since reuseport_lock is not taken, sk is accessed under
         * rcu_read_lock()
         */
        rcu_read_lock();
        for (i = 0; i < map->max_entries; i++) {
                sk = rcu_dereference(array->ptrs[i]);
                if (sk) {
                        write_lock_bh(&sk->sk_callback_lock);
                        /*
                         * No need for WRITE_ONCE(). At this point,
                         * no one is reading it without taking the
                         * sk->sk_callback_lock.
                         */
                        sk->sk_user_data = NULL;
                        write_unlock_bh(&sk->sk_callback_lock);
                        RCU_INIT_POINTER(array->ptrs[i], NULL);
                }
        }
        rcu_read_unlock();

        /*
         * Once reaching here, all sk->sk_user_data is not
         * referencing this "array". "array" can be freed now.
         */
        bpf_map_area_free(array);
}

static struct bpf_map *reuseport_array_alloc(union bpf_attr *attr)
{
        int numa_node = bpf_map_attr_numa_node(attr);
        struct reuseport_array *array;

        /* allocate all map elements and zero-initialize them */
        array = bpf_map_area_alloc(struct_size(array, ptrs, attr->max_entries), numa_node);
        if (!array)
                return ERR_PTR(-ENOMEM);

        /* copy mandatory map attributes */
        bpf_map_init_from_attr(&array->map, attr);

        return &array->map;
}

int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
                                       void *value)
{
        struct sock *sk;
        int err;

        if (map->value_size != sizeof(u64))
                return -ENOSPC;

        rcu_read_lock();
        sk = reuseport_array_lookup_elem(map, key);
        if (sk) {
                *(u64 *)value = __sock_gen_cookie(sk);
                err = 0;
        } else {
                err = -ENOENT;
        }
        rcu_read_unlock();

        return err;
}

static int
reuseport_array_update_check(const struct reuseport_array *array,
                             const struct sock *nsk,
                             const struct sock *osk,
                             const struct sock_reuseport *nsk_reuse,
                             u32 map_flags)
{
        if (osk && map_flags == BPF_NOEXIST)
                return -EEXIST;

        if (!osk && map_flags == BPF_EXIST)
                return -ENOENT;

        if (nsk->sk_protocol != IPPROTO_UDP && nsk->sk_protocol != IPPROTO_TCP)
                return -ENOTSUPP;

        if (nsk->sk_family != AF_INET && nsk->sk_family != AF_INET6)
                return -ENOTSUPP;

        if (nsk->sk_type != SOCK_STREAM && nsk->sk_type != SOCK_DGRAM)
                return -ENOTSUPP;

        /*
         * sk must be hashed (i.e. listening in the TCP case or binded
         * in the UDP case) and
         * it must also be a SO_REUSEPORT sk (i.e. reuse cannot be NULL).
         *
         * Also, sk will be used in bpf helper that is protected by
         * rcu_read_lock().
         */
        if (!sock_flag(nsk, SOCK_RCU_FREE) || !sk_hashed(nsk) || !nsk_reuse)
                return -EINVAL;

        /* READ_ONCE because the sk->sk_callback_lock may not be held here */
        if (READ_ONCE(nsk->sk_user_data))
                return -EBUSY;

        return 0;
}

/*
 * Called from syscall only.
 * The "nsk" in the fd refcnt.
 * The "osk" and "reuse" are protected by reuseport_lock.
 */
int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
                                       void *value, u64 map_flags)
{
        struct reuseport_array *array = reuseport_array(map);
        struct sock *free_osk = NULL, *osk, *nsk;
        struct sock_reuseport *reuse;
        u32 index = *(u32 *)key;
        uintptr_t sk_user_data;
        struct socket *socket;
        int err, fd;

        if (map_flags > BPF_EXIST)
                return -EINVAL;

        if (index >= map->max_entries)
                return -E2BIG;

        if (map->value_size == sizeof(u64)) {
                u64 fd64 = *(u64 *)value;

                if (fd64 > S32_MAX)
                        return -EINVAL;
                fd = fd64;
        } else {
                fd = *(int *)value;
        }

        socket = sockfd_lookup(fd, &err);
        if (!socket)
                return err;

        nsk = socket->sk;
        if (!nsk) {
                err = -EINVAL;
                goto put_file;
        }

        /* Quick checks before taking reuseport_lock */
        err = reuseport_array_update_check(array, nsk,
                                           rcu_access_pointer(array->ptrs[index]),
                                           rcu_access_pointer(nsk->sk_reuseport_cb),
                                           map_flags);
        if (err)
                goto put_file;

        spin_lock_bh(&reuseport_lock);
        /*
         * Some of the checks only need reuseport_lock
         * but it is done under sk_callback_lock also
         * for simplicity reason.
         */
        write_lock_bh(&nsk->sk_callback_lock);

        osk = rcu_dereference_protected(array->ptrs[index],
                                        lockdep_is_held(&reuseport_lock));
        reuse = rcu_dereference_protected(nsk->sk_reuseport_cb,
                                          lockdep_is_held(&reuseport_lock));
        err = reuseport_array_update_check(array, nsk, osk, reuse, map_flags);
        if (err)
                goto put_file_unlock;

        sk_user_data = (uintptr_t)&array->ptrs[index] | SK_USER_DATA_NOCOPY |
                SK_USER_DATA_BPF;
        WRITE_ONCE(nsk->sk_user_data, (void *)sk_user_data);
        rcu_assign_pointer(array->ptrs[index], nsk);
        free_osk = osk;
        err = 0;

put_file_unlock:
        write_unlock_bh(&nsk->sk_callback_lock);

        if (free_osk) {
                write_lock_bh(&free_osk->sk_callback_lock);
                WRITE_ONCE(free_osk->sk_user_data, NULL);
                write_unlock_bh(&free_osk->sk_callback_lock);
        }

        spin_unlock_bh(&reuseport_lock);
put_file:
        sockfd_put(socket);
        return err;
}

/* Called from syscall */
static int reuseport_array_get_next_key(struct bpf_map *map, void *key,
                                        void *next_key)
{
        struct reuseport_array *array = reuseport_array(map);
        u32 index = key ? *(u32 *)key : U32_MAX;
        u32 *next = (u32 *)next_key;

        if (index >= array->map.max_entries) {
                *next = 0;
                return 0;
        }

        if (index == array->map.max_entries - 1)
                return -ENOENT;

        *next = index + 1;
        return 0;
}

static u64 reuseport_array_mem_usage(const struct bpf_map *map)
{
        struct reuseport_array *array;

        return struct_size(array, ptrs, map->max_entries);
}

BTF_ID_LIST_SINGLE(reuseport_array_map_btf_ids, struct, reuseport_array)
const struct bpf_map_ops reuseport_array_ops = {
        .map_meta_equal = bpf_map_meta_equal,
        .map_alloc_check = reuseport_array_alloc_check,
        .map_alloc = reuseport_array_alloc,
        .map_free = reuseport_array_free,
        .map_lookup_elem = reuseport_array_lookup_elem,
        .map_get_next_key = reuseport_array_get_next_key,
        .map_delete_elem = reuseport_array_delete_elem,
        .map_mem_usage = reuseport_array_mem_usage,
        .map_btf_id = &reuseport_array_map_btf_ids[0],
};




















 2778 












 2599 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_BH_H
#define _LINUX_BH_H

#include <linux/instruction_pointer.h>
#include <linux/preempt.h>

#if defined(CONFIG_PREEMPT_RT) || defined(CONFIG_TRACE_IRQFLAGS)
extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt);
#else
static __always_inline void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
{
        preempt_count_add(cnt);
        barrier();
}
#endif

static inline void local_bh_disable(void)
{
        __local_bh_disable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET);
}

extern void _local_bh_enable(void);
extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt);

static inline void local_bh_enable_ip(unsigned long ip)
{
        __local_bh_enable_ip(ip, SOFTIRQ_DISABLE_OFFSET);
}

static inline void local_bh_enable(void)
{
        __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET);
}

#ifdef CONFIG_PREEMPT_RT
extern bool local_bh_blocked(void);
#else
static inline bool local_bh_blocked(void) { return false; }
#endif

#endif /* _LINUX_BH_H */
































































































































    2 





    3 









































    3 





















































   23 





   56 








   23 






   56 









































  288 




   10 
  296 
    2 






   59 








   59 






   26 












    8 

























   99 










  243 




  228 
  228 




   54 
   54 



   54 





  194 






  180 




   23 
   23 
   23 
   23 




  186 
   13 
   13 
   13 











   43 


   11 













   17 

   26 










    4 



   50 



































   33 





   33 






   33 














   20 

   37 

















  158 
   10 








   12 
   64 











   40 


   40 






   40 



    6 






    9 




  313 


  313 
   35 
  309 
  291 
   47 

  310 

  314 







   12 
   46 









    6 
   10 






  151 




  151 
  151 
  151 





  146 
   10 


  150 












  194 






  194 
  194 






















  102 
  103 



   74 
    5 




   68 
   17 





  103 
  103 




  194 
   55 
    3 





  183 
   13 

  194 














  193 
    4 















  166 
  166 















   47 

























   14 
   14 


   14 
   14 









   20 






   20 


    6 

   15 



   15 

   11 
    3 









   15 











































    4 










    8 














    3 
    2 
    4 




















   16 




    1 



   10 

    1 


    4 







    1 


    2 


















































































  164 










































  180 










































  180 

  180 










  137 


    1 




   33 

   35 







   47 






   21 





  135 






  170 


    7 


  164 














   99 


























   38 



   63 





    5 


   51 





   48 








    2 














   40 
   15 



   26 








  135 



   99 

   40 





   13 











    2 




















   11 

   11 


















   11 































    3 








    2 

    2 











   23 








    6 







   13 




















    1 






   23 

   22 



   21 




    2 




    3 




   66 




    1 

    4 

    2 




   36 

   24 





   10 
    6 




   33 








   51 




   23 




   88 






    3 



    2 







   38 

    3 





   12 




   29 









    3 



    3 















   16 


   44 
    8 

   52 

   52 

    5 
   47 


   36 
   11 



   52 


   16 





    6 
    6 





   60 












    1 
   60 







   49 










    4 







   44 

    2 



   45 













   45 













   18 













    6 






    2 





   12 


   12 










    3 













    4 


































   14 



































   36 




  111 




  111 



  111 






  111 
  111 














   12 





   12 



    4 










    3 
    1 







    1 
    3 





    4 









   47 






   18 






   29 




   30 



   16 









   45 













   23 






   23 













   12 







    9 



  275 




  268 
    7 



  275 




  348 


  349 




  329 
   23 







  355 


















  286 
    2 


  285 









   19 









    4 






  271 



  250 
   20 









   37 
    7 










  302 









  281 
   35 





    3 



  282 
    9 



   54 







  292 

    4 




    3 



  270 
   23 










   29 










    4 


  288 

   51 



    9 
   43 




  275 
    4 






   55 













    2 



   14 









    1 




   13 


   12 














    1 



    1 




    2 



  227 




   53 

  174 







   49 
  167 





   47 










   10 















  208 
    2 


















  178 











   30 







    4 












    2 










   23 



























   71 








   58 
   19 




   33 


    4 
   37 






    2 





    3 





    3 


   64 





    7 



   31 












    6 




















    7 





   30 









   37 


















   27 

    6 

   24 










    1 
    2 




   25 
   25 




  100 



   99 
    2 


    1 


   97 
    3 






   34 


    1 


   33 






  361 
   11 






  371 









    1 


  369 







  347 

   42 





    2 

   55 





   35 

    2 



  343 




   13 
  330 






   17 

  315 
   20 


    3 


  324 
   17 



  342 


   36 
    5 

   41 














  302 

  295 
    7 

  342 












  340 








  371 


































   60 
   54 
   10 
   12 
   17 



   54 







   55 




















   16 










    5 








    3 



    6 


    2 








    3 
    8 





    2 


    9 









   68 




   17 
    9 









    6 



    2 


    5 




    1 
    4 



    1 
   11 

























































  117 
















    1 







   16 




  104 
  104 










  103 












   65 




   68 
   19 
    6 





   10 






   71 


   18 



    8 






   60 


   18 

   18 



   46 











    4 







    6 

   51 

   14 




   23 


   41 









    4 










   44 

   42 
    2 



   26 














   17 
    2 

   19 

    2 







    9 

   13 








   68 





   18 








   56 























   93 















   93 





   14 








   23 











   22 

    1 





   16 



    1 












   11 
    4 



   11 




   10 









    3 

    7 



   11 





   13 



    1 



    6 



   10 



    5 
    3 








   13 
    1 








    1 


    1 


    1 



    1 








    1 




  374 




  356 

    1 



    8 



    7 


    5 











    2 





    1 

    1 


















  366 





   48 





   48 















   48 







   48 







   48 







  122 






  122 





  122 












  122 







  122 


    2 
   14 



   39 
    8 






   38 


   47 



















   20 

   15 












   20 


   20 

















    2 







   14 



   20 
   20 


   16 



   20 



   20 

   14 



   20 


   18 
    9 



   20 


   20 

















    7 




    7 


    2 




    8 
    7 






























































































































































































































  199 























  199 














































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * NET4:        Implementation of BSD Unix domain sockets.
 *
 * Authors:        Alan Cox, <alan@lxorguk.ukuu.org.uk>
 *
 * Fixes:
 *                Linus Torvalds        :        Assorted bug cures.
 *                Niibe Yutaka        :        async I/O support.
 *                Carsten Paeth        :        PF_UNIX check, address fixes.
 *                Alan Cox        :        Limit size of allocated blocks.
 *                Alan Cox        :        Fixed the stupid socketpair bug.
 *                Alan Cox        :        BSD compatibility fine tuning.
 *                Alan Cox        :        Fixed a bug in connect when interrupted.
 *                Alan Cox        :        Sorted out a proper draft version of
 *                                        file descriptor passing hacked up from
 *                                        Mike Shaver's work.
 *                Marty Leisner        :        Fixes to fd passing
 *                Nick Nevin        :        recvmsg bugfix.
 *                Alan Cox        :        Started proper garbage collector
 *                Heiko EiBfeldt        :        Missing verify_area check
 *                Alan Cox        :        Started POSIXisms
 *                Andreas Schwab        :        Replace inode by dentry for proper
 *                                        reference counting
 *                Kirk Petersen        :        Made this a module
 *            Christoph Rohland        :        Elegant non-blocking accept/connect algorithm.
 *                                        Lots of bug fixes.
 *             Alexey Kuznetosv        :        Repaired (I hope) bugs introduces
 *                                        by above two patches.
 *             Andrea Arcangeli        :        If possible we block in connect(2)
 *                                        if the max backlog of the listen socket
 *                                        is been reached. This won't break
 *                                        old apps and it will avoid huge amount
 *                                        of socks hashed (this for unix_gc()
 *                                        performances reasons).
 *                                        Security fix that limits the max
 *                                        number of socks to 2*max_files and
 *                                        the number of skb queueable in the
 *                                        dgram receiver.
 *                Artur Skawina   :        Hash function optimizations
 *             Alexey Kuznetsov   :        Full scale SMP. Lot of bugs are introduced 8)
 *              Malcolm Beattie   :        Set peercred for socketpair
 *             Michal Ostrowski   :       Module initialization cleanup.
 *             Arnaldo C. Melo        :        Remove MOD_{INC,DEC}_USE_COUNT,
 *                                             the core infrastructure is doing that
 *                                             for all net proto families now (2.5.69+)
 *
 * Known differences from reference BSD that was tested:
 *
 *        [TO FIX]
 *        ECONNREFUSED is not returned from one end of a connected() socket to the
 *                other the moment one end closes.
 *        fstat() doesn't return st_dev=0, and give the blksize as high water mark
 *                and a fake inode identifier (nor the BSD first socket fstat twice bug).
 *        [NOT TO FIX]
 *        accept() returns a path name even if the connecting socket has closed
 *                in the meantime (BSD loses the path and gives up).
 *        accept() returns 0 length path for an unbound connector. BSD returns 16
 *                and a null first byte in the path (but not for gethost/peername - BSD bug ??)
 *        socketpair(...SOCK_RAW..) doesn't panic the kernel.
 *        BSD af_unix apparently has connect forgetting to block properly.
 *                (need to check this with the POSIX spec in detail)
 *
 * Differences from 2.0.0-11-... (ANK)
 *        Bug fixes and improvements.
 *                - client shutdown killed server socket.
 *                - removed all useless cli/sti pairs.
 *
 *        Semantic changes/extensions.
 *                - generic control message passing.
 *                - SCM_CREDENTIALS control message.
 *                - "Abstract" (not FS based) socket bindings.
 *                  Abstract names are sequences of bytes (not zero terminated)
 *                  started by 0, so that this name space does not intersect
 *                  with BSD names.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/bpf-cgroup.h>
#include <linux/btf_ids.h>
#include <linux/dcache.h>
#include <linux/errno.h>
#include <linux/fcntl.h>
#include <linux/file.h>
#include <linux/filter.h>
#include <linux/fs.h>
#include <linux/fs_struct.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/mount.h>
#include <linux/namei.h>
#include <linux/net.h>
#include <linux/pidfs.h>
#include <linux/poll.h>
#include <linux/proc_fs.h>
#include <linux/sched/signal.h>
#include <linux/security.h>
#include <linux/seq_file.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/socket.h>
#include <linux/splice.h>
#include <linux/string.h>
#include <linux/uaccess.h>
#include <net/af_unix.h>
#include <net/net_namespace.h>
#include <net/scm.h>
#include <net/tcp_states.h>
#include <uapi/linux/sockios.h>
#include <uapi/linux/termios.h>

#include "af_unix.h"

static atomic_long_t unix_nr_socks;
static struct hlist_head bsd_socket_buckets[UNIX_HASH_SIZE / 2];
static spinlock_t bsd_socket_locks[UNIX_HASH_SIZE / 2];

/* SMP locking strategy:
 *    hash table is protected with spinlock.
 *    each socket state is protected by separate spinlock.
 */
#ifdef CONFIG_PROVE_LOCKING
#define cmp_ptr(l, r)        (((l) > (r)) - ((l) < (r)))

static int unix_table_lock_cmp_fn(const struct lockdep_map *a,
                                  const struct lockdep_map *b)
{
        return cmp_ptr(a, b);
}

static int unix_state_lock_cmp_fn(const struct lockdep_map *_a,
                                  const struct lockdep_map *_b)
{
        const struct unix_sock *a, *b;

        a = container_of(_a, struct unix_sock, lock.dep_map);
        b = container_of(_b, struct unix_sock, lock.dep_map);

        if (a->sk.sk_state == TCP_LISTEN) {
                /* unix_stream_connect(): Before the 2nd unix_state_lock(),
                 *
                 *   1. a is TCP_LISTEN.
                 *   2. b is not a.
                 *   3. concurrent connect(b -> a) must fail.
                 *
                 * Except for 2. & 3., the b's state can be any possible
                 * value due to concurrent connect() or listen().
                 *
                 * 2. is detected in debug_spin_lock_before(), and 3. cannot
                 * be expressed as lock_cmp_fn.
                 */
                switch (b->sk.sk_state) {
                case TCP_CLOSE:
                case TCP_ESTABLISHED:
                case TCP_LISTEN:
                        return -1;
                default:
                        /* Invalid case. */
                        return 0;
                }
        }

        /* Should never happen.  Just to be symmetric. */
        if (b->sk.sk_state == TCP_LISTEN) {
                switch (b->sk.sk_state) {
                case TCP_CLOSE:
                case TCP_ESTABLISHED:
                        return 1;
                default:
                        return 0;
                }
        }

        /* unix_state_double_lock(): ascending address order. */
        return cmp_ptr(a, b);
}

static int unix_recvq_lock_cmp_fn(const struct lockdep_map *_a,
                                  const struct lockdep_map *_b)
{
        const struct sock *a, *b;

        a = container_of(_a, struct sock, sk_receive_queue.lock.dep_map);
        b = container_of(_b, struct sock, sk_receive_queue.lock.dep_map);

        /* unix_collect_skb(): listener -> embryo order. */
        if (a->sk_state == TCP_LISTEN && unix_sk(b)->listener == a)
                return -1;

        /* Should never happen.  Just to be symmetric. */
        if (b->sk_state == TCP_LISTEN && unix_sk(a)->listener == b)
                return 1;

        return 0;
}
#endif

static unsigned int unix_unbound_hash(struct sock *sk)
{
        unsigned long hash = (unsigned long)sk;

        hash ^= hash >> 16;
        hash ^= hash >> 8;
        hash ^= sk->sk_type;

        return hash & UNIX_HASH_MOD;
}

static unsigned int unix_bsd_hash(struct inode *i)
{
        return i->i_ino & UNIX_HASH_MOD;
}

static unsigned int unix_abstract_hash(struct sockaddr_un *sunaddr,
                                       int addr_len, int type)
{
        __wsum csum = csum_partial(sunaddr, addr_len, 0);
        unsigned int hash;

        hash = (__force unsigned int)csum_fold(csum);
        hash ^= hash >> 8;
        hash ^= type;

        return UNIX_HASH_MOD + 1 + (hash & UNIX_HASH_MOD);
}

static void unix_table_double_lock(struct net *net,
                                   unsigned int hash1, unsigned int hash2)
{
        if (hash1 == hash2) {
                spin_lock(&net->unx.table.locks[hash1]);
                return;
        }

        if (hash1 > hash2)
                swap(hash1, hash2);

        spin_lock(&net->unx.table.locks[hash1]);
        spin_lock(&net->unx.table.locks[hash2]);
}

static void unix_table_double_unlock(struct net *net,
                                     unsigned int hash1, unsigned int hash2)
{
        if (hash1 == hash2) {
                spin_unlock(&net->unx.table.locks[hash1]);
                return;
        }

        spin_unlock(&net->unx.table.locks[hash1]);
        spin_unlock(&net->unx.table.locks[hash2]);
}

#ifdef CONFIG_SECURITY_NETWORK
static void unix_get_secdata(struct scm_cookie *scm, struct sk_buff *skb)
{
        UNIXCB(skb).secid = scm->secid;
}

static inline void unix_set_secdata(struct scm_cookie *scm, struct sk_buff *skb)
{
        scm->secid = UNIXCB(skb).secid;
}

static inline bool unix_secdata_eq(struct scm_cookie *scm, struct sk_buff *skb)
{
        return (scm->secid == UNIXCB(skb).secid);
}
#else
static inline void unix_get_secdata(struct scm_cookie *scm, struct sk_buff *skb)
{ }

static inline void unix_set_secdata(struct scm_cookie *scm, struct sk_buff *skb)
{ }

static inline bool unix_secdata_eq(struct scm_cookie *scm, struct sk_buff *skb)
{
        return true;
}
#endif /* CONFIG_SECURITY_NETWORK */

static inline int unix_may_send(struct sock *sk, struct sock *osk)
{
        return !unix_peer(osk) || unix_peer(osk) == sk;
}

static inline int unix_recvq_full_lockless(const struct sock *sk)
{
        return skb_queue_len_lockless(&sk->sk_receive_queue) > sk->sk_max_ack_backlog;
}

struct sock *unix_peer_get(struct sock *s)
{
        struct sock *peer;

        unix_state_lock(s);
        peer = unix_peer(s);
        if (peer)
                sock_hold(peer);
        unix_state_unlock(s);
        return peer;
}
EXPORT_SYMBOL_GPL(unix_peer_get);

static struct unix_address *unix_create_addr(struct sockaddr_un *sunaddr,
                                             int addr_len)
{
        struct unix_address *addr;

        addr = kmalloc(sizeof(*addr) + addr_len, GFP_KERNEL);
        if (!addr)
                return NULL;

        refcount_set(&addr->refcnt, 1);
        addr->len = addr_len;
        memcpy(addr->name, sunaddr, addr_len);

        return addr;
}

static inline void unix_release_addr(struct unix_address *addr)
{
        if (refcount_dec_and_test(&addr->refcnt))
                kfree(addr);
}

/*
 *        Check unix socket name:
 *                - should be not zero length.
 *                - if started by not zero, should be NULL terminated (FS object)
 *                - if started by zero, it is abstract name.
 */

static int unix_validate_addr(struct sockaddr_un *sunaddr, int addr_len)
{
        if (addr_len <= offsetof(struct sockaddr_un, sun_path) ||
            addr_len > sizeof(*sunaddr))
                return -EINVAL;

        if (sunaddr->sun_family != AF_UNIX)
                return -EINVAL;

        return 0;
}

static int unix_mkname_bsd(struct sockaddr_un *sunaddr, int addr_len)
{
        struct sockaddr_storage *addr = (struct sockaddr_storage *)sunaddr;
        short offset = offsetof(struct sockaddr_storage, __data);

        BUILD_BUG_ON(offset != offsetof(struct sockaddr_un, sun_path));

        /* This may look like an off by one error but it is a bit more
         * subtle.  108 is the longest valid AF_UNIX path for a binding.
         * sun_path[108] doesn't as such exist.  However in kernel space
         * we are guaranteed that it is a valid memory location in our
         * kernel address buffer because syscall functions always pass
         * a pointer of struct sockaddr_storage which has a bigger buffer
         * than 108.  Also, we must terminate sun_path for strlen() in
         * getname_kernel().
         */
        addr->__data[addr_len - offset] = 0;

        /* Don't pass sunaddr->sun_path to strlen().  Otherwise, 108 will
         * cause panic if CONFIG_FORTIFY_SOURCE=y.  Let __fortify_strlen()
         * know the actual buffer.
         */
        return strlen(addr->__data) + offset + 1;
}

static void __unix_remove_socket(struct sock *sk)
{
        sk_del_node_init(sk);
}

static void __unix_insert_socket(struct net *net, struct sock *sk)
{
        DEBUG_NET_WARN_ON_ONCE(!sk_unhashed(sk));
        sk_add_node(sk, &net->unx.table.buckets[sk->sk_hash]);
}

static void __unix_set_addr_hash(struct net *net, struct sock *sk,
                                 struct unix_address *addr, unsigned int hash)
{
        __unix_remove_socket(sk);
        smp_store_release(&unix_sk(sk)->addr, addr);

        sk->sk_hash = hash;
        __unix_insert_socket(net, sk);
}

static void unix_remove_socket(struct net *net, struct sock *sk)
{
        spin_lock(&net->unx.table.locks[sk->sk_hash]);
        __unix_remove_socket(sk);
        spin_unlock(&net->unx.table.locks[sk->sk_hash]);
}

static void unix_insert_unbound_socket(struct net *net, struct sock *sk)
{
        spin_lock(&net->unx.table.locks[sk->sk_hash]);
        __unix_insert_socket(net, sk);
        spin_unlock(&net->unx.table.locks[sk->sk_hash]);
}

static void unix_insert_bsd_socket(struct sock *sk)
{
        spin_lock(&bsd_socket_locks[sk->sk_hash]);
        sk_add_bind_node(sk, &bsd_socket_buckets[sk->sk_hash]);
        spin_unlock(&bsd_socket_locks[sk->sk_hash]);
}

static void unix_remove_bsd_socket(struct sock *sk)
{
        if (!hlist_unhashed(&sk->sk_bind_node)) {
                spin_lock(&bsd_socket_locks[sk->sk_hash]);
                __sk_del_bind_node(sk);
                spin_unlock(&bsd_socket_locks[sk->sk_hash]);

                sk_node_init(&sk->sk_bind_node);
        }
}

static struct sock *__unix_find_socket_byname(struct net *net,
                                              struct sockaddr_un *sunname,
                                              int len, unsigned int hash)
{
        struct sock *s;

        sk_for_each(s, &net->unx.table.buckets[hash]) {
                struct unix_sock *u = unix_sk(s);

                if (u->addr->len == len &&
                    !memcmp(u->addr->name, sunname, len))
                        return s;
        }
        return NULL;
}

static inline struct sock *unix_find_socket_byname(struct net *net,
                                                   struct sockaddr_un *sunname,
                                                   int len, unsigned int hash)
{
        struct sock *s;

        spin_lock(&net->unx.table.locks[hash]);
        s = __unix_find_socket_byname(net, sunname, len, hash);
        if (s)
                sock_hold(s);
        spin_unlock(&net->unx.table.locks[hash]);
        return s;
}

static struct sock *unix_find_socket_byinode(struct inode *i)
{
        unsigned int hash = unix_bsd_hash(i);
        struct sock *s;

        spin_lock(&bsd_socket_locks[hash]);
        sk_for_each_bound(s, &bsd_socket_buckets[hash]) {
                struct dentry *dentry = unix_sk(s)->path.dentry;

                if (dentry && d_backing_inode(dentry) == i) {
                        sock_hold(s);
                        spin_unlock(&bsd_socket_locks[hash]);
                        return s;
                }
        }
        spin_unlock(&bsd_socket_locks[hash]);
        return NULL;
}

/* Support code for asymmetrically connected dgram sockets
 *
 * If a datagram socket is connected to a socket not itself connected
 * to the first socket (eg, /dev/log), clients may only enqueue more
 * messages if the present receive queue of the server socket is not
 * "too large". This means there's a second writeability condition
 * poll and sendmsg need to test. The dgram recv code will do a wake
 * up on the peer_wait wait queue of a socket upon reception of a
 * datagram which needs to be propagated to sleeping would-be writers
 * since these might not have sent anything so far. This can't be
 * accomplished via poll_wait because the lifetime of the server
 * socket might be less than that of its clients if these break their
 * association with it or if the server socket is closed while clients
 * are still connected to it and there's no way to inform "a polling
 * implementation" that it should let go of a certain wait queue
 *
 * In order to propagate a wake up, a wait_queue_entry_t of the client
 * socket is enqueued on the peer_wait queue of the server socket
 * whose wake function does a wake_up on the ordinary client socket
 * wait queue. This connection is established whenever a write (or
 * poll for write) hit the flow control condition and broken when the
 * association to the server socket is dissolved or after a wake up
 * was relayed.
 */

static int unix_dgram_peer_wake_relay(wait_queue_entry_t *q, unsigned mode, int flags,
                                      void *key)
{
        struct unix_sock *u;
        wait_queue_head_t *u_sleep;

        u = container_of(q, struct unix_sock, peer_wake);

        __remove_wait_queue(&unix_sk(u->peer_wake.private)->peer_wait,
                            q);
        u->peer_wake.private = NULL;

        /* relaying can only happen while the wq still exists */
        u_sleep = sk_sleep(&u->sk);
        if (u_sleep)
                wake_up_interruptible_poll(u_sleep, key_to_poll(key));

        return 0;
}

static int unix_dgram_peer_wake_connect(struct sock *sk, struct sock *other)
{
        struct unix_sock *u, *u_other;
        int rc;

        u = unix_sk(sk);
        u_other = unix_sk(other);
        rc = 0;
        spin_lock(&u_other->peer_wait.lock);

        if (!u->peer_wake.private) {
                u->peer_wake.private = other;
                __add_wait_queue(&u_other->peer_wait, &u->peer_wake);

                rc = 1;
        }

        spin_unlock(&u_other->peer_wait.lock);
        return rc;
}

static void unix_dgram_peer_wake_disconnect(struct sock *sk,
                                            struct sock *other)
{
        struct unix_sock *u, *u_other;

        u = unix_sk(sk);
        u_other = unix_sk(other);
        spin_lock(&u_other->peer_wait.lock);

        if (u->peer_wake.private == other) {
                __remove_wait_queue(&u_other->peer_wait, &u->peer_wake);
                u->peer_wake.private = NULL;
        }

        spin_unlock(&u_other->peer_wait.lock);
}

static void unix_dgram_peer_wake_disconnect_wakeup(struct sock *sk,
                                                   struct sock *other)
{
        unix_dgram_peer_wake_disconnect(sk, other);
        wake_up_interruptible_poll(sk_sleep(sk),
                                   EPOLLOUT |
                                   EPOLLWRNORM |
                                   EPOLLWRBAND);
}

/* preconditions:
 *        - unix_peer(sk) == other
 *        - association is stable
 */
static int unix_dgram_peer_wake_me(struct sock *sk, struct sock *other)
{
        int connected;

        connected = unix_dgram_peer_wake_connect(sk, other);

        /* If other is SOCK_DEAD, we want to make sure we signal
         * POLLOUT, such that a subsequent write() can get a
         * -ECONNREFUSED. Otherwise, if we haven't queued any skbs
         * to other and its full, we will hang waiting for POLLOUT.
         */
        if (unix_recvq_full_lockless(other) && !sock_flag(other, SOCK_DEAD))
                return 1;

        if (connected)
                unix_dgram_peer_wake_disconnect(sk, other);

        return 0;
}

static int unix_writable(const struct sock *sk, unsigned char state)
{
        return state != TCP_LISTEN &&
                (refcount_read(&sk->sk_wmem_alloc) << 2) <= READ_ONCE(sk->sk_sndbuf);
}

static void unix_write_space(struct sock *sk)
{
        struct socket_wq *wq;

        rcu_read_lock();
        if (unix_writable(sk, READ_ONCE(sk->sk_state))) {
                wq = rcu_dereference(sk->sk_wq);
                if (skwq_has_sleeper(wq))
                        wake_up_interruptible_sync_poll(&wq->wait,
                                EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
                sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
        }
        rcu_read_unlock();
}

/* When dgram socket disconnects (or changes its peer), we clear its receive
 * queue of packets arrived from previous peer. First, it allows to do
 * flow control based only on wmem_alloc; second, sk connected to peer
 * may receive messages only from that peer. */
static void unix_dgram_disconnected(struct sock *sk, struct sock *other)
{
        if (!skb_queue_empty(&sk->sk_receive_queue)) {
                skb_queue_purge_reason(&sk->sk_receive_queue,
                                       SKB_DROP_REASON_UNIX_DISCONNECT);

                wake_up_interruptible_all(&unix_sk(sk)->peer_wait);

                /* If one link of bidirectional dgram pipe is disconnected,
                 * we signal error. Messages are lost. Do not make this,
                 * when peer was not connected to us.
                 */
                if (!sock_flag(other, SOCK_DEAD) && unix_peer(other) == sk) {
                        WRITE_ONCE(other->sk_err, ECONNRESET);
                        sk_error_report(other);
                }
        }
}

static void unix_sock_destructor(struct sock *sk)
{
        struct unix_sock *u = unix_sk(sk);

        skb_queue_purge_reason(&sk->sk_receive_queue, SKB_DROP_REASON_SOCKET_CLOSE);

        DEBUG_NET_WARN_ON_ONCE(refcount_read(&sk->sk_wmem_alloc));
        DEBUG_NET_WARN_ON_ONCE(!sk_unhashed(sk));
        DEBUG_NET_WARN_ON_ONCE(sk->sk_socket);
        if (!sock_flag(sk, SOCK_DEAD)) {
                pr_info("Attempt to release alive unix socket: %p\n", sk);
                return;
        }

        if (u->addr)
                unix_release_addr(u->addr);

        atomic_long_dec(&unix_nr_socks);
        sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
#ifdef UNIX_REFCNT_DEBUG
        pr_debug("UNIX %p is destroyed, %ld are still alive.\n", sk,
                atomic_long_read(&unix_nr_socks));
#endif
}

static unsigned int unix_skb_len(const struct sk_buff *skb)
{
        return skb->len - UNIXCB(skb).consumed;
}

static void unix_release_sock(struct sock *sk, int embrion)
{
        struct unix_sock *u = unix_sk(sk);
        struct sock *skpair;
        struct sk_buff *skb;
        struct path path;
        int state;

        unix_remove_socket(sock_net(sk), sk);
        unix_remove_bsd_socket(sk);

        /* Clear state */
        unix_state_lock(sk);
        sock_orphan(sk);
        WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
        path             = u->path;
        u->path.dentry = NULL;
        u->path.mnt = NULL;
        state = sk->sk_state;
        WRITE_ONCE(sk->sk_state, TCP_CLOSE);

        skpair = unix_peer(sk);
        unix_peer(sk) = NULL;

        unix_state_unlock(sk);

#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
        u->oob_skb = NULL;
#endif

        wake_up_interruptible_all(&u->peer_wait);

        if (skpair != NULL) {
                if (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) {
                        struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);

#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
                        if (skb && !unix_skb_len(skb))
                                skb = skb_peek_next(skb, &sk->sk_receive_queue);
#endif
                        unix_state_lock(skpair);
                        /* No more writes */
                        WRITE_ONCE(skpair->sk_shutdown, SHUTDOWN_MASK);
                        if (skb || embrion)
                                WRITE_ONCE(skpair->sk_err, ECONNRESET);
                        unix_state_unlock(skpair);
                        skpair->sk_state_change(skpair);
                        sk_wake_async(skpair, SOCK_WAKE_WAITD, POLL_HUP);
                }

                unix_dgram_peer_wake_disconnect(sk, skpair);
                sock_put(skpair); /* It may now die */
        }

        /* Try to flush out this socket. Throw out buffers at least */

        while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) {
                if (state == TCP_LISTEN)
                        unix_release_sock(skb->sk, 1);

                /* passed fds are erased in the kfree_skb hook */
                kfree_skb_reason(skb, SKB_DROP_REASON_SOCKET_CLOSE);
        }

        if (path.dentry)
                path_put(&path);

        sock_put(sk);

        /* ---- Socket is dead now and most probably destroyed ---- */

        /*
         * Fixme: BSD difference: In BSD all sockets connected to us get
         *          ECONNRESET and we die on the spot. In Linux we behave
         *          like files and pipes do and wait for the last
         *          dereference.
         *
         * Can't we simply set sock->err?
         *
         *          What the above comment does talk about? --ANK(980817)
         */

        if (READ_ONCE(unix_tot_inflight))
                unix_gc();                /* Garbage collect fds */
}

struct unix_peercred {
        struct pid *peer_pid;
        const struct cred *peer_cred;
};

static inline int prepare_peercred(struct unix_peercred *peercred)
{
        struct pid *pid;
        int err;

        pid = task_tgid(current);
        err = pidfs_register_pid(pid);
        if (likely(!err)) {
                peercred->peer_pid = get_pid(pid);
                peercred->peer_cred = get_current_cred();
        }
        return err;
}

static void drop_peercred(struct unix_peercred *peercred)
{
        const struct cred *cred = NULL;
        struct pid *pid = NULL;

        might_sleep();

        swap(peercred->peer_pid, pid);
        swap(peercred->peer_cred, cred);

        put_pid(pid);
        put_cred(cred);
}

static inline void init_peercred(struct sock *sk,
                                 const struct unix_peercred *peercred)
{
        sk->sk_peer_pid = peercred->peer_pid;
        sk->sk_peer_cred = peercred->peer_cred;
}

static void update_peercred(struct sock *sk, struct unix_peercred *peercred)
{
        const struct cred *old_cred;
        struct pid *old_pid;

        spin_lock(&sk->sk_peer_lock);
        old_pid = sk->sk_peer_pid;
        old_cred = sk->sk_peer_cred;
        init_peercred(sk, peercred);
        spin_unlock(&sk->sk_peer_lock);

        peercred->peer_pid = old_pid;
        peercred->peer_cred = old_cred;
}

static void copy_peercred(struct sock *sk, struct sock *peersk)
{
        lockdep_assert_held(&unix_sk(peersk)->lock);

        spin_lock(&sk->sk_peer_lock);
        sk->sk_peer_pid = get_pid(peersk->sk_peer_pid);
        sk->sk_peer_cred = get_cred(peersk->sk_peer_cred);
        spin_unlock(&sk->sk_peer_lock);
}

static bool unix_may_passcred(const struct sock *sk)
{
        return sk->sk_scm_credentials || sk->sk_scm_pidfd;
}

static int unix_listen(struct socket *sock, int backlog)
{
        int err;
        struct sock *sk = sock->sk;
        struct unix_sock *u = unix_sk(sk);
        struct unix_peercred peercred = {};

        err = -EOPNOTSUPP;
        if (sock->type != SOCK_STREAM && sock->type != SOCK_SEQPACKET)
                goto out;        /* Only stream/seqpacket sockets accept */
        err = -EINVAL;
        if (!READ_ONCE(u->addr))
                goto out;        /* No listens on an unbound socket */
        err = prepare_peercred(&peercred);
        if (err)
                goto out;
        unix_state_lock(sk);
        if (sk->sk_state != TCP_CLOSE && sk->sk_state != TCP_LISTEN)
                goto out_unlock;
        if (backlog > sk->sk_max_ack_backlog)
                wake_up_interruptible_all(&u->peer_wait);
        sk->sk_max_ack_backlog        = backlog;
        WRITE_ONCE(sk->sk_state, TCP_LISTEN);

        /* set credentials so connect can copy them */
        update_peercred(sk, &peercred);
        err = 0;

out_unlock:
        unix_state_unlock(sk);
        drop_peercred(&peercred);
out:
        return err;
}

static int unix_release(struct socket *);
static int unix_bind(struct socket *, struct sockaddr *, int);
static int unix_stream_connect(struct socket *, struct sockaddr *,
                               int addr_len, int flags);
static int unix_socketpair(struct socket *, struct socket *);
static int unix_accept(struct socket *, struct socket *, struct proto_accept_arg *arg);
static int unix_getname(struct socket *, struct sockaddr *, int);
static __poll_t unix_poll(struct file *, struct socket *, poll_table *);
static __poll_t unix_dgram_poll(struct file *, struct socket *,
                                    poll_table *);
static int unix_ioctl(struct socket *, unsigned int, unsigned long);
#ifdef CONFIG_COMPAT
static int unix_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg);
#endif
static int unix_shutdown(struct socket *, int);
static int unix_stream_sendmsg(struct socket *, struct msghdr *, size_t);
static int unix_stream_recvmsg(struct socket *, struct msghdr *, size_t, int);
static ssize_t unix_stream_splice_read(struct socket *,  loff_t *ppos,
                                       struct pipe_inode_info *, size_t size,
                                       unsigned int flags);
static int unix_dgram_sendmsg(struct socket *, struct msghdr *, size_t);
static int unix_dgram_recvmsg(struct socket *, struct msghdr *, size_t, int);
static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
static int unix_dgram_connect(struct socket *, struct sockaddr *,
                              int, int);
static int unix_seqpacket_sendmsg(struct socket *, struct msghdr *, size_t);
static int unix_seqpacket_recvmsg(struct socket *, struct msghdr *, size_t,
                                  int);

#ifdef CONFIG_PROC_FS
static int unix_count_nr_fds(struct sock *sk)
{
        struct sk_buff *skb;
        struct unix_sock *u;
        int nr_fds = 0;

        spin_lock(&sk->sk_receive_queue.lock);
        skb = skb_peek(&sk->sk_receive_queue);
        while (skb) {
                u = unix_sk(skb->sk);
                nr_fds += atomic_read(&u->scm_stat.nr_fds);
                skb = skb_peek_next(skb, &sk->sk_receive_queue);
        }
        spin_unlock(&sk->sk_receive_queue.lock);

        return nr_fds;
}

static void unix_show_fdinfo(struct seq_file *m, struct socket *sock)
{
        struct sock *sk = sock->sk;
        unsigned char s_state;
        struct unix_sock *u;
        int nr_fds = 0;

        if (sk) {
                s_state = READ_ONCE(sk->sk_state);
                u = unix_sk(sk);

                /* SOCK_STREAM and SOCK_SEQPACKET sockets never change their
                 * sk_state after switching to TCP_ESTABLISHED or TCP_LISTEN.
                 * SOCK_DGRAM is ordinary. So, no lock is needed.
                 */
                if (sock->type == SOCK_DGRAM || s_state == TCP_ESTABLISHED)
                        nr_fds = atomic_read(&u->scm_stat.nr_fds);
                else if (s_state == TCP_LISTEN)
                        nr_fds = unix_count_nr_fds(sk);

                seq_printf(m, "scm_fds: %u\n", nr_fds);
        }
}
#else
#define unix_show_fdinfo NULL
#endif

static bool unix_custom_sockopt(int optname)
{
        switch (optname) {
        case SO_INQ:
                return true;
        default:
                return false;
        }
}

static int unix_setsockopt(struct socket *sock, int level, int optname,
                           sockptr_t optval, unsigned int optlen)
{
        struct unix_sock *u = unix_sk(sock->sk);
        struct sock *sk = sock->sk;
        int val;

        if (level != SOL_SOCKET)
                return -EOPNOTSUPP;

        if (!unix_custom_sockopt(optname))
                return sock_setsockopt(sock, level, optname, optval, optlen);

        if (optlen != sizeof(int))
                return -EINVAL;

        if (copy_from_sockptr(&val, optval, sizeof(val)))
                return -EFAULT;

        switch (optname) {
        case SO_INQ:
                if (sk->sk_type != SOCK_STREAM)
                        return -EINVAL;

                if (val > 1 || val < 0)
                        return -EINVAL;

                WRITE_ONCE(u->recvmsg_inq, val);
                break;
        default:
                return -ENOPROTOOPT;
        }

        return 0;
}

static const struct proto_ops unix_stream_ops = {
        .family =        PF_UNIX,
        .owner =        THIS_MODULE,
        .release =        unix_release,
        .bind =                unix_bind,
        .connect =        unix_stream_connect,
        .socketpair =        unix_socketpair,
        .accept =        unix_accept,
        .getname =        unix_getname,
        .poll =                unix_poll,
        .ioctl =        unix_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl =        unix_compat_ioctl,
#endif
        .listen =        unix_listen,
        .shutdown =        unix_shutdown,
        .setsockopt =        unix_setsockopt,
        .sendmsg =        unix_stream_sendmsg,
        .recvmsg =        unix_stream_recvmsg,
        .read_skb =        unix_stream_read_skb,
        .mmap =                sock_no_mmap,
        .splice_read =        unix_stream_splice_read,
        .set_peek_off =        sk_set_peek_off,
        .show_fdinfo =        unix_show_fdinfo,
};

static const struct proto_ops unix_dgram_ops = {
        .family =        PF_UNIX,
        .owner =        THIS_MODULE,
        .release =        unix_release,
        .bind =                unix_bind,
        .connect =        unix_dgram_connect,
        .socketpair =        unix_socketpair,
        .accept =        sock_no_accept,
        .getname =        unix_getname,
        .poll =                unix_dgram_poll,
        .ioctl =        unix_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl =        unix_compat_ioctl,
#endif
        .listen =        sock_no_listen,
        .shutdown =        unix_shutdown,
        .sendmsg =        unix_dgram_sendmsg,
        .read_skb =        unix_read_skb,
        .recvmsg =        unix_dgram_recvmsg,
        .mmap =                sock_no_mmap,
        .set_peek_off =        sk_set_peek_off,
        .show_fdinfo =        unix_show_fdinfo,
};

static const struct proto_ops unix_seqpacket_ops = {
        .family =        PF_UNIX,
        .owner =        THIS_MODULE,
        .release =        unix_release,
        .bind =                unix_bind,
        .connect =        unix_stream_connect,
        .socketpair =        unix_socketpair,
        .accept =        unix_accept,
        .getname =        unix_getname,
        .poll =                unix_dgram_poll,
        .ioctl =        unix_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl =        unix_compat_ioctl,
#endif
        .listen =        unix_listen,
        .shutdown =        unix_shutdown,
        .sendmsg =        unix_seqpacket_sendmsg,
        .recvmsg =        unix_seqpacket_recvmsg,
        .mmap =                sock_no_mmap,
        .set_peek_off =        sk_set_peek_off,
        .show_fdinfo =        unix_show_fdinfo,
};

static void unix_close(struct sock *sk, long timeout)
{
        /* Nothing to do here, unix socket does not need a ->close().
         * This is merely for sockmap.
         */
}

static bool unix_bpf_bypass_getsockopt(int level, int optname)
{
        if (level == SOL_SOCKET) {
                switch (optname) {
                case SO_PEERPIDFD:
                        return true;
                default:
                        return false;
                }
        }

        return false;
}

struct proto unix_dgram_proto = {
        .name                        = "UNIX",
        .owner                        = THIS_MODULE,
        .obj_size                = sizeof(struct unix_sock),
        .close                        = unix_close,
        .bpf_bypass_getsockopt        = unix_bpf_bypass_getsockopt,
#ifdef CONFIG_BPF_SYSCALL
        .psock_update_sk_prot        = unix_dgram_bpf_update_proto,
#endif
};

struct proto unix_stream_proto = {
        .name                        = "UNIX-STREAM",
        .owner                        = THIS_MODULE,
        .obj_size                = sizeof(struct unix_sock),
        .close                        = unix_close,
        .bpf_bypass_getsockopt        = unix_bpf_bypass_getsockopt,
#ifdef CONFIG_BPF_SYSCALL
        .psock_update_sk_prot        = unix_stream_bpf_update_proto,
#endif
};

static struct sock *unix_create1(struct net *net, struct socket *sock, int kern, int type)
{
        struct unix_sock *u;
        struct sock *sk;
        int err;

        atomic_long_inc(&unix_nr_socks);
        if (atomic_long_read(&unix_nr_socks) > 2 * get_max_files()) {
                err = -ENFILE;
                goto err;
        }

        if (type == SOCK_STREAM)
                sk = sk_alloc(net, PF_UNIX, GFP_KERNEL, &unix_stream_proto, kern);
        else /*dgram and  seqpacket */
                sk = sk_alloc(net, PF_UNIX, GFP_KERNEL, &unix_dgram_proto, kern);

        if (!sk) {
                err = -ENOMEM;
                goto err;
        }

        sock_init_data(sock, sk);

        sk->sk_scm_rights        = 1;
        sk->sk_hash                = unix_unbound_hash(sk);
        sk->sk_allocation        = GFP_KERNEL_ACCOUNT;
        sk->sk_write_space        = unix_write_space;
        sk->sk_max_ack_backlog        = READ_ONCE(net->unx.sysctl_max_dgram_qlen);
        sk->sk_destruct                = unix_sock_destructor;
        lock_set_cmp_fn(&sk->sk_receive_queue.lock, unix_recvq_lock_cmp_fn, NULL);

        u = unix_sk(sk);
        u->listener = NULL;
        u->vertex = NULL;
        u->path.dentry = NULL;
        u->path.mnt = NULL;
        spin_lock_init(&u->lock);
        lock_set_cmp_fn(&u->lock, unix_state_lock_cmp_fn, NULL);
        mutex_init(&u->iolock); /* single task reading lock */
        mutex_init(&u->bindlock); /* single task binding lock */
        init_waitqueue_head(&u->peer_wait);
        init_waitqueue_func_entry(&u->peer_wake, unix_dgram_peer_wake_relay);
        memset(&u->scm_stat, 0, sizeof(struct scm_stat));
        unix_insert_unbound_socket(net, sk);

        sock_prot_inuse_add(net, sk->sk_prot, 1);

        return sk;

err:
        atomic_long_dec(&unix_nr_socks);
        return ERR_PTR(err);
}

static int unix_create(struct net *net, struct socket *sock, int protocol,
                       int kern)
{
        struct sock *sk;

        if (protocol && protocol != PF_UNIX)
                return -EPROTONOSUPPORT;

        sock->state = SS_UNCONNECTED;

        switch (sock->type) {
        case SOCK_STREAM:
                set_bit(SOCK_CUSTOM_SOCKOPT, &sock->flags);
                sock->ops = &unix_stream_ops;
                break;
                /*
                 *        Believe it or not BSD has AF_UNIX, SOCK_RAW though
                 *        nothing uses it.
                 */
        case SOCK_RAW:
                sock->type = SOCK_DGRAM;
                fallthrough;
        case SOCK_DGRAM:
                sock->ops = &unix_dgram_ops;
                break;
        case SOCK_SEQPACKET:
                sock->ops = &unix_seqpacket_ops;
                break;
        default:
                return -ESOCKTNOSUPPORT;
        }

        sk = unix_create1(net, sock, kern, sock->type);
        if (IS_ERR(sk))
                return PTR_ERR(sk);

        return 0;
}

static int unix_release(struct socket *sock)
{
        struct sock *sk = sock->sk;

        if (!sk)
                return 0;

        sk->sk_prot->close(sk, 0);
        unix_release_sock(sk, 0);
        sock->sk = NULL;

        return 0;
}

static struct sock *unix_find_bsd(struct sockaddr_un *sunaddr, int addr_len,
                                  int type, int flags)
{
        struct inode *inode;
        struct path path;
        struct sock *sk;
        int err;

        unix_mkname_bsd(sunaddr, addr_len);

        if (flags & SOCK_COREDUMP) {
                const struct cred *cred;
                struct cred *kcred;
                struct path root;

                kcred = prepare_kernel_cred(&init_task);
                if (!kcred) {
                        err = -ENOMEM;
                        goto fail;
                }

                task_lock(&init_task);
                get_fs_root(init_task.fs, &root);
                task_unlock(&init_task);

                cred = override_creds(kcred);
                err = vfs_path_lookup(root.dentry, root.mnt, sunaddr->sun_path,
                                      LOOKUP_BENEATH | LOOKUP_NO_SYMLINKS |
                                      LOOKUP_NO_MAGICLINKS, &path);
                put_cred(revert_creds(cred));
                path_put(&root);
                if (err)
                        goto fail;
        } else {
                err = kern_path(sunaddr->sun_path, LOOKUP_FOLLOW, &path);
                if (err)
                        goto fail;

                err = path_permission(&path, MAY_WRITE);
                if (err)
                        goto path_put;
        }

        err = -ECONNREFUSED;
        inode = d_backing_inode(path.dentry);
        if (!S_ISSOCK(inode->i_mode))
                goto path_put;

        sk = unix_find_socket_byinode(inode);
        if (!sk)
                goto path_put;

        err = -EPROTOTYPE;
        if (sk->sk_type == type)
                touch_atime(&path);
        else
                goto sock_put;

        path_put(&path);

        return sk;

sock_put:
        sock_put(sk);
path_put:
        path_put(&path);
fail:
        return ERR_PTR(err);
}

static struct sock *unix_find_abstract(struct net *net,
                                       struct sockaddr_un *sunaddr,
                                       int addr_len, int type)
{
        unsigned int hash = unix_abstract_hash(sunaddr, addr_len, type);
        struct dentry *dentry;
        struct sock *sk;

        sk = unix_find_socket_byname(net, sunaddr, addr_len, hash);
        if (!sk)
                return ERR_PTR(-ECONNREFUSED);

        dentry = unix_sk(sk)->path.dentry;
        if (dentry)
                touch_atime(&unix_sk(sk)->path);

        return sk;
}

static struct sock *unix_find_other(struct net *net,
                                    struct sockaddr_un *sunaddr,
                                    int addr_len, int type, int flags)
{
        struct sock *sk;

        if (sunaddr->sun_path[0])
                sk = unix_find_bsd(sunaddr, addr_len, type, flags);
        else
                sk = unix_find_abstract(net, sunaddr, addr_len, type);

        return sk;
}

static int unix_autobind(struct sock *sk)
{
        struct unix_sock *u = unix_sk(sk);
        unsigned int new_hash, old_hash;
        struct net *net = sock_net(sk);
        struct unix_address *addr;
        u32 lastnum, ordernum;
        int err;

        err = mutex_lock_interruptible(&u->bindlock);
        if (err)
                return err;

        if (u->addr)
                goto out;

        err = -ENOMEM;
        addr = kzalloc(sizeof(*addr) +
                       offsetof(struct sockaddr_un, sun_path) + 16, GFP_KERNEL);
        if (!addr)
                goto out;

        addr->len = offsetof(struct sockaddr_un, sun_path) + 6;
        addr->name->sun_family = AF_UNIX;
        refcount_set(&addr->refcnt, 1);

        old_hash = sk->sk_hash;
        ordernum = get_random_u32();
        lastnum = ordernum & 0xFFFFF;
retry:
        ordernum = (ordernum + 1) & 0xFFFFF;
        sprintf(addr->name->sun_path + 1, "%05x", ordernum);

        new_hash = unix_abstract_hash(addr->name, addr->len, sk->sk_type);
        unix_table_double_lock(net, old_hash, new_hash);

        if (__unix_find_socket_byname(net, addr->name, addr->len, new_hash)) {
                unix_table_double_unlock(net, old_hash, new_hash);

                /* __unix_find_socket_byname() may take long time if many names
                 * are already in use.
                 */
                cond_resched();

                if (ordernum == lastnum) {
                        /* Give up if all names seems to be in use. */
                        err = -ENOSPC;
                        unix_release_addr(addr);
                        goto out;
                }

                goto retry;
        }

        __unix_set_addr_hash(net, sk, addr, new_hash);
        unix_table_double_unlock(net, old_hash, new_hash);
        err = 0;

out:        mutex_unlock(&u->bindlock);
        return err;
}

static int unix_bind_bsd(struct sock *sk, struct sockaddr_un *sunaddr,
                         int addr_len)
{
        umode_t mode = S_IFSOCK |
               (SOCK_INODE(sk->sk_socket)->i_mode & ~current_umask());
        struct unix_sock *u = unix_sk(sk);
        unsigned int new_hash, old_hash;
        struct net *net = sock_net(sk);
        struct mnt_idmap *idmap;
        struct unix_address *addr;
        struct dentry *dentry;
        struct path parent;
        int err;

        addr_len = unix_mkname_bsd(sunaddr, addr_len);
        addr = unix_create_addr(sunaddr, addr_len);
        if (!addr)
                return -ENOMEM;

        /*
         * Get the parent directory, calculate the hash for last
         * component.
         */
        dentry = start_creating_path(AT_FDCWD, addr->name->sun_path, &parent, 0);
        if (IS_ERR(dentry)) {
                err = PTR_ERR(dentry);
                goto out;
        }

        /*
         * All right, let's create it.
         */
        idmap = mnt_idmap(parent.mnt);
        err = security_path_mknod(&parent, dentry, mode, 0);
        if (!err)
                err = vfs_mknod(idmap, d_inode(parent.dentry), dentry, mode, 0);
        if (err)
                goto out_path;
        err = mutex_lock_interruptible(&u->bindlock);
        if (err)
                goto out_unlink;
        if (u->addr)
                goto out_unlock;

        old_hash = sk->sk_hash;
        new_hash = unix_bsd_hash(d_backing_inode(dentry));
        unix_table_double_lock(net, old_hash, new_hash);
        u->path.mnt = mntget(parent.mnt);
        u->path.dentry = dget(dentry);
        __unix_set_addr_hash(net, sk, addr, new_hash);
        unix_table_double_unlock(net, old_hash, new_hash);
        unix_insert_bsd_socket(sk);
        mutex_unlock(&u->bindlock);
        end_creating_path(&parent, dentry);
        return 0;

out_unlock:
        mutex_unlock(&u->bindlock);
        err = -EINVAL;
out_unlink:
        /* failed after successful mknod?  unlink what we'd created... */
        vfs_unlink(idmap, d_inode(parent.dentry), dentry, NULL);
out_path:
        end_creating_path(&parent, dentry);
out:
        unix_release_addr(addr);
        return err == -EEXIST ? -EADDRINUSE : err;
}

static int unix_bind_abstract(struct sock *sk, struct sockaddr_un *sunaddr,
                              int addr_len)
{
        struct unix_sock *u = unix_sk(sk);
        unsigned int new_hash, old_hash;
        struct net *net = sock_net(sk);
        struct unix_address *addr;
        int err;

        addr = unix_create_addr(sunaddr, addr_len);
        if (!addr)
                return -ENOMEM;

        err = mutex_lock_interruptible(&u->bindlock);
        if (err)
                goto out;

        if (u->addr) {
                err = -EINVAL;
                goto out_mutex;
        }

        old_hash = sk->sk_hash;
        new_hash = unix_abstract_hash(addr->name, addr->len, sk->sk_type);
        unix_table_double_lock(net, old_hash, new_hash);

        if (__unix_find_socket_byname(net, addr->name, addr->len, new_hash))
                goto out_spin;

        __unix_set_addr_hash(net, sk, addr, new_hash);
        unix_table_double_unlock(net, old_hash, new_hash);
        mutex_unlock(&u->bindlock);
        return 0;

out_spin:
        unix_table_double_unlock(net, old_hash, new_hash);
        err = -EADDRINUSE;
out_mutex:
        mutex_unlock(&u->bindlock);
out:
        unix_release_addr(addr);
        return err;
}

static int unix_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
{
        struct sockaddr_un *sunaddr = (struct sockaddr_un *)uaddr;
        struct sock *sk = sock->sk;
        int err;

        if (addr_len == offsetof(struct sockaddr_un, sun_path) &&
            sunaddr->sun_family == AF_UNIX)
                return unix_autobind(sk);

        err = unix_validate_addr(sunaddr, addr_len);
        if (err)
                return err;

        if (sunaddr->sun_path[0])
                err = unix_bind_bsd(sk, sunaddr, addr_len);
        else
                err = unix_bind_abstract(sk, sunaddr, addr_len);

        return err;
}

static void unix_state_double_lock(struct sock *sk1, struct sock *sk2)
{
        if (unlikely(sk1 == sk2) || !sk2) {
                unix_state_lock(sk1);
                return;
        }

        if (sk1 > sk2)
                swap(sk1, sk2);

        unix_state_lock(sk1);
        unix_state_lock(sk2);
}

static void unix_state_double_unlock(struct sock *sk1, struct sock *sk2)
{
        if (unlikely(sk1 == sk2) || !sk2) {
                unix_state_unlock(sk1);
                return;
        }
        unix_state_unlock(sk1);
        unix_state_unlock(sk2);
}

static int unix_dgram_connect(struct socket *sock, struct sockaddr *addr,
                              int alen, int flags)
{
        struct sockaddr_un *sunaddr = (struct sockaddr_un *)addr;
        struct sock *sk = sock->sk;
        struct sock *other;
        int err;

        err = -EINVAL;
        if (alen < offsetofend(struct sockaddr, sa_family))
                goto out;

        if (addr->sa_family != AF_UNSPEC) {
                err = unix_validate_addr(sunaddr, alen);
                if (err)
                        goto out;

                err = BPF_CGROUP_RUN_PROG_UNIX_CONNECT_LOCK(sk, addr, &alen);
                if (err)
                        goto out;

                if (unix_may_passcred(sk) && !READ_ONCE(unix_sk(sk)->addr)) {
                        err = unix_autobind(sk);
                        if (err)
                                goto out;
                }

restart:
                other = unix_find_other(sock_net(sk), sunaddr, alen, sock->type, 0);
                if (IS_ERR(other)) {
                        err = PTR_ERR(other);
                        goto out;
                }

                unix_state_double_lock(sk, other);

                /* Apparently VFS overslept socket death. Retry. */
                if (sock_flag(other, SOCK_DEAD)) {
                        unix_state_double_unlock(sk, other);
                        sock_put(other);
                        goto restart;
                }

                err = -EPERM;
                if (!unix_may_send(sk, other))
                        goto out_unlock;

                err = security_unix_may_send(sk->sk_socket, other->sk_socket);
                if (err)
                        goto out_unlock;

                WRITE_ONCE(sk->sk_state, TCP_ESTABLISHED);
                WRITE_ONCE(other->sk_state, TCP_ESTABLISHED);
        } else {
                /*
                 *        1003.1g breaking connected state with AF_UNSPEC
                 */
                other = NULL;
                unix_state_double_lock(sk, other);
        }

        /*
         * If it was connected, reconnect.
         */
        if (unix_peer(sk)) {
                struct sock *old_peer = unix_peer(sk);

                unix_peer(sk) = other;
                if (!other)
                        WRITE_ONCE(sk->sk_state, TCP_CLOSE);
                unix_dgram_peer_wake_disconnect_wakeup(sk, old_peer);

                unix_state_double_unlock(sk, other);

                if (other != old_peer) {
                        unix_dgram_disconnected(sk, old_peer);

                        unix_state_lock(old_peer);
                        if (!unix_peer(old_peer))
                                WRITE_ONCE(old_peer->sk_state, TCP_CLOSE);
                        unix_state_unlock(old_peer);
                }

                sock_put(old_peer);
        } else {
                unix_peer(sk) = other;
                unix_state_double_unlock(sk, other);
        }

        return 0;

out_unlock:
        unix_state_double_unlock(sk, other);
        sock_put(other);
out:
        return err;
}

static long unix_wait_for_peer(struct sock *other, long timeo)
{
        struct unix_sock *u = unix_sk(other);
        int sched;
        DEFINE_WAIT(wait);

        prepare_to_wait_exclusive(&u->peer_wait, &wait, TASK_INTERRUPTIBLE);

        sched = !sock_flag(other, SOCK_DEAD) &&
                !(other->sk_shutdown & RCV_SHUTDOWN) &&
                unix_recvq_full_lockless(other);

        unix_state_unlock(other);

        if (sched)
                timeo = schedule_timeout(timeo);

        finish_wait(&u->peer_wait, &wait);
        return timeo;
}

static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr,
                               int addr_len, int flags)
{
        struct sockaddr_un *sunaddr = (struct sockaddr_un *)uaddr;
        struct sock *sk = sock->sk, *newsk = NULL, *other = NULL;
        struct unix_sock *u = unix_sk(sk), *newu, *otheru;
        struct unix_peercred peercred = {};
        struct net *net = sock_net(sk);
        struct sk_buff *skb = NULL;
        unsigned char state;
        long timeo;
        int err;

        err = unix_validate_addr(sunaddr, addr_len);
        if (err)
                goto out;

        err = BPF_CGROUP_RUN_PROG_UNIX_CONNECT_LOCK(sk, uaddr, &addr_len);
        if (err)
                goto out;

        if (unix_may_passcred(sk) && !READ_ONCE(u->addr)) {
                err = unix_autobind(sk);
                if (err)
                        goto out;
        }

        timeo = sock_sndtimeo(sk, flags & O_NONBLOCK);

        /* First of all allocate resources.
         * If we will make it after state is locked,
         * we will have to recheck all again in any case.
         */

        /* create new sock for complete connection */
        newsk = unix_create1(net, NULL, 0, sock->type);
        if (IS_ERR(newsk)) {
                err = PTR_ERR(newsk);
                goto out;
        }

        err = prepare_peercred(&peercred);
        if (err)
                goto out;

        /* Allocate skb for sending to listening sock */
        skb = sock_wmalloc(newsk, 1, 0, GFP_KERNEL);
        if (!skb) {
                err = -ENOMEM;
                goto out_free_sk;
        }

restart:
        /*  Find listening sock. */
        other = unix_find_other(net, sunaddr, addr_len, sk->sk_type, flags);
        if (IS_ERR(other)) {
                err = PTR_ERR(other);
                goto out_free_skb;
        }

        unix_state_lock(other);

        /* Apparently VFS overslept socket death. Retry. */
        if (sock_flag(other, SOCK_DEAD)) {
                unix_state_unlock(other);
                sock_put(other);
                goto restart;
        }

        if (other->sk_state != TCP_LISTEN ||
            other->sk_shutdown & RCV_SHUTDOWN) {
                err = -ECONNREFUSED;
                goto out_unlock;
        }

        if (unix_recvq_full_lockless(other)) {
                if (!timeo) {
                        err = -EAGAIN;
                        goto out_unlock;
                }

                timeo = unix_wait_for_peer(other, timeo);
                sock_put(other);

                err = sock_intr_errno(timeo);
                if (signal_pending(current))
                        goto out_free_skb;

                goto restart;
        }

        /* self connect and simultaneous connect are eliminated
         * by rejecting TCP_LISTEN socket to avoid deadlock.
         */
        state = READ_ONCE(sk->sk_state);
        if (unlikely(state != TCP_CLOSE)) {
                err = state == TCP_ESTABLISHED ? -EISCONN : -EINVAL;
                goto out_unlock;
        }

        unix_state_lock(sk);

        if (unlikely(sk->sk_state != TCP_CLOSE)) {
                err = sk->sk_state == TCP_ESTABLISHED ? -EISCONN : -EINVAL;
                unix_state_unlock(sk);
                goto out_unlock;
        }

        err = security_unix_stream_connect(sk, other, newsk);
        if (err) {
                unix_state_unlock(sk);
                goto out_unlock;
        }

        /* The way is open! Fastly set all the necessary fields... */

        sock_hold(sk);
        unix_peer(newsk) = sk;
        newsk->sk_state = TCP_ESTABLISHED;
        newsk->sk_type = sk->sk_type;
        newsk->sk_scm_recv_flags = other->sk_scm_recv_flags;
        init_peercred(newsk, &peercred);

        newu = unix_sk(newsk);
        newu->listener = other;
        RCU_INIT_POINTER(newsk->sk_wq, &newu->peer_wq);
        otheru = unix_sk(other);

        /* copy address information from listening to new sock
         *
         * The contents of *(otheru->addr) and otheru->path
         * are seen fully set up here, since we have found
         * otheru in hash under its lock.  Insertion into the
         * hash chain we'd found it in had been done in an
         * earlier critical area protected by the chain's lock,
         * the same one where we'd set *(otheru->addr) contents,
         * as well as otheru->path and otheru->addr itself.
         *
         * Using smp_store_release() here to set newu->addr
         * is enough to make those stores, as well as stores
         * to newu->path visible to anyone who gets newu->addr
         * by smp_load_acquire().  IOW, the same warranties
         * as for unix_sock instances bound in unix_bind() or
         * in unix_autobind().
         */
        if (otheru->path.dentry) {
                path_get(&otheru->path);
                newu->path = otheru->path;
        }
        refcount_inc(&otheru->addr->refcnt);
        smp_store_release(&newu->addr, otheru->addr);

        /* Set credentials */
        copy_peercred(sk, other);

        sock->state        = SS_CONNECTED;
        WRITE_ONCE(sk->sk_state, TCP_ESTABLISHED);
        sock_hold(newsk);

        smp_mb__after_atomic();        /* sock_hold() does an atomic_inc() */
        unix_peer(sk)        = newsk;

        unix_state_unlock(sk);

        /* take ten and send info to listening sock */
        spin_lock(&other->sk_receive_queue.lock);
        __skb_queue_tail(&other->sk_receive_queue, skb);
        spin_unlock(&other->sk_receive_queue.lock);
        unix_state_unlock(other);
        other->sk_data_ready(other);
        sock_put(other);
        return 0;

out_unlock:
        unix_state_unlock(other);
        sock_put(other);
out_free_skb:
        consume_skb(skb);
out_free_sk:
        unix_release_sock(newsk, 0);
out:
        drop_peercred(&peercred);
        return err;
}

static int unix_socketpair(struct socket *socka, struct socket *sockb)
{
        struct unix_peercred ska_peercred = {}, skb_peercred = {};
        struct sock *ska = socka->sk, *skb = sockb->sk;
        int err;

        err = prepare_peercred(&ska_peercred);
        if (err)
                return err;

        err = prepare_peercred(&skb_peercred);
        if (err) {
                drop_peercred(&ska_peercred);
                return err;
        }

        /* Join our sockets back to back */
        sock_hold(ska);
        sock_hold(skb);
        unix_peer(ska) = skb;
        unix_peer(skb) = ska;
        init_peercred(ska, &ska_peercred);
        init_peercred(skb, &skb_peercred);

        ska->sk_state = TCP_ESTABLISHED;
        skb->sk_state = TCP_ESTABLISHED;
        socka->state  = SS_CONNECTED;
        sockb->state  = SS_CONNECTED;
        return 0;
}

static int unix_accept(struct socket *sock, struct socket *newsock,
                       struct proto_accept_arg *arg)
{
        struct sock *sk = sock->sk;
        struct sk_buff *skb;
        struct sock *tsk;

        arg->err = -EOPNOTSUPP;
        if (sock->type != SOCK_STREAM && sock->type != SOCK_SEQPACKET)
                goto out;

        arg->err = -EINVAL;
        if (READ_ONCE(sk->sk_state) != TCP_LISTEN)
                goto out;

        /* If socket state is TCP_LISTEN it cannot change (for now...),
         * so that no locks are necessary.
         */

        skb = skb_recv_datagram(sk, (arg->flags & O_NONBLOCK) ? MSG_DONTWAIT : 0,
                                &arg->err);
        if (!skb) {
                /* This means receive shutdown. */
                if (arg->err == 0)
                        arg->err = -EINVAL;
                goto out;
        }

        tsk = skb->sk;
        skb_free_datagram(sk, skb);
        wake_up_interruptible(&unix_sk(sk)->peer_wait);

        if (tsk->sk_type == SOCK_STREAM)
                set_bit(SOCK_CUSTOM_SOCKOPT, &newsock->flags);

        /* attach accepted sock to socket */
        unix_state_lock(tsk);
        unix_update_edges(unix_sk(tsk));
        newsock->state = SS_CONNECTED;
        sock_graft(tsk, newsock);
        unix_state_unlock(tsk);
        return 0;

out:
        return arg->err;
}


static int unix_getname(struct socket *sock, struct sockaddr *uaddr, int peer)
{
        struct sock *sk = sock->sk;
        struct unix_address *addr;
        DECLARE_SOCKADDR(struct sockaddr_un *, sunaddr, uaddr);
        int err = 0;

        if (peer) {
                sk = unix_peer_get(sk);

                err = -ENOTCONN;
                if (!sk)
                        goto out;
                err = 0;
        } else {
                sock_hold(sk);
        }

        addr = smp_load_acquire(&unix_sk(sk)->addr);
        if (!addr) {
                sunaddr->sun_family = AF_UNIX;
                sunaddr->sun_path[0] = 0;
                err = offsetof(struct sockaddr_un, sun_path);
        } else {
                err = addr->len;
                memcpy(sunaddr, addr->name, addr->len);

                if (peer)
                        BPF_CGROUP_RUN_SA_PROG(sk, uaddr, &err,
                                               CGROUP_UNIX_GETPEERNAME);
                else
                        BPF_CGROUP_RUN_SA_PROG(sk, uaddr, &err,
                                               CGROUP_UNIX_GETSOCKNAME);
        }
        sock_put(sk);
out:
        return err;
}

/* The "user->unix_inflight" variable is protected by the garbage
 * collection lock, and we just read it locklessly here. If you go
 * over the limit, there might be a tiny race in actually noticing
 * it across threads. Tough.
 */
static inline bool too_many_unix_fds(struct task_struct *p)
{
        struct user_struct *user = current_user();

        if (unlikely(READ_ONCE(user->unix_inflight) > task_rlimit(p, RLIMIT_NOFILE)))
                return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
        return false;
}

static int unix_attach_fds(struct scm_cookie *scm, struct sk_buff *skb)
{
        if (too_many_unix_fds(current))
                return -ETOOMANYREFS;

        UNIXCB(skb).fp = scm->fp;
        scm->fp = NULL;

        if (unix_prepare_fpl(UNIXCB(skb).fp))
                return -ENOMEM;

        return 0;
}

static void unix_detach_fds(struct scm_cookie *scm, struct sk_buff *skb)
{
        scm->fp = UNIXCB(skb).fp;
        UNIXCB(skb).fp = NULL;

        unix_destroy_fpl(scm->fp);
}

static void unix_peek_fds(struct scm_cookie *scm, struct sk_buff *skb)
{
        scm->fp = scm_fp_dup(UNIXCB(skb).fp);
}

static void unix_destruct_scm(struct sk_buff *skb)
{
        struct scm_cookie scm;

        memset(&scm, 0, sizeof(scm));
        scm.pid = UNIXCB(skb).pid;
        if (UNIXCB(skb).fp)
                unix_detach_fds(&scm, skb);

        /* Alas, it calls VFS */
        /* So fscking what? fput() had been SMP-safe since the last Summer */
        scm_destroy(&scm);
        sock_wfree(skb);
}

static int unix_scm_to_skb(struct scm_cookie *scm, struct sk_buff *skb, bool send_fds)
{
        int err = 0;

        UNIXCB(skb).pid = get_pid(scm->pid);
        UNIXCB(skb).uid = scm->creds.uid;
        UNIXCB(skb).gid = scm->creds.gid;
        UNIXCB(skb).fp = NULL;
        unix_get_secdata(scm, skb);
        if (scm->fp && send_fds)
                err = unix_attach_fds(scm, skb);

        skb->destructor = unix_destruct_scm;
        return err;
}

static void unix_skb_to_scm(struct sk_buff *skb, struct scm_cookie *scm)
{
        scm_set_cred(scm, UNIXCB(skb).pid, UNIXCB(skb).uid, UNIXCB(skb).gid);
        unix_set_secdata(scm, skb);
}

/**
 * unix_maybe_add_creds() - Adds current task uid/gid and struct pid to skb if needed.
 * @skb: skb to attach creds to.
 * @sk: Sender sock.
 * @other: Receiver sock.
 *
 * Some apps rely on write() giving SCM_CREDENTIALS
 * We include credentials if source or destination socket
 * asserted SOCK_PASSCRED.
 *
 * Context: May sleep.
 * Return: On success zero, on error a negative error code is returned.
 */
static int unix_maybe_add_creds(struct sk_buff *skb, const struct sock *sk,
                                const struct sock *other)
{
        if (UNIXCB(skb).pid)
                return 0;

        if (unix_may_passcred(sk) || unix_may_passcred(other) ||
            !other->sk_socket) {
                struct pid *pid;
                int err;

                pid = task_tgid(current);
                err = pidfs_register_pid(pid);
                if (unlikely(err))
                        return err;

                UNIXCB(skb).pid = get_pid(pid);
                current_uid_gid(&UNIXCB(skb).uid, &UNIXCB(skb).gid);
        }

        return 0;
}

static bool unix_skb_scm_eq(struct sk_buff *skb,
                            struct scm_cookie *scm)
{
        return UNIXCB(skb).pid == scm->pid &&
               uid_eq(UNIXCB(skb).uid, scm->creds.uid) &&
               gid_eq(UNIXCB(skb).gid, scm->creds.gid) &&
               unix_secdata_eq(scm, skb);
}

static void scm_stat_add(struct sock *sk, struct sk_buff *skb)
{
        struct scm_fp_list *fp = UNIXCB(skb).fp;
        struct unix_sock *u = unix_sk(sk);

        if (unlikely(fp && fp->count)) {
                atomic_add(fp->count, &u->scm_stat.nr_fds);
                unix_add_edges(fp, u);
        }
}

static void scm_stat_del(struct sock *sk, struct sk_buff *skb)
{
        struct scm_fp_list *fp = UNIXCB(skb).fp;
        struct unix_sock *u = unix_sk(sk);

        if (unlikely(fp && fp->count)) {
                atomic_sub(fp->count, &u->scm_stat.nr_fds);
                unix_del_edges(fp);
        }
}

/*
 *        Send AF_UNIX data.
 */

static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg,
                              size_t len)
{
        struct sock *sk = sock->sk, *other = NULL;
        struct unix_sock *u = unix_sk(sk);
        struct scm_cookie scm;
        struct sk_buff *skb;
        int data_len = 0;
        int sk_locked;
        long timeo;
        int err;

        err = scm_send(sock, msg, &scm, false);
        if (err < 0)
                return err;

        wait_for_unix_gc(scm.fp);

        if (msg->msg_flags & MSG_OOB) {
                err = -EOPNOTSUPP;
                goto out;
        }

        if (msg->msg_namelen) {
                err = unix_validate_addr(msg->msg_name, msg->msg_namelen);
                if (err)
                        goto out;

                err = BPF_CGROUP_RUN_PROG_UNIX_SENDMSG_LOCK(sk,
                                                            msg->msg_name,
                                                            &msg->msg_namelen,
                                                            NULL);
                if (err)
                        goto out;
        }

        if (unix_may_passcred(sk) && !READ_ONCE(u->addr)) {
                err = unix_autobind(sk);
                if (err)
                        goto out;
        }

        if (len > READ_ONCE(sk->sk_sndbuf) - 32) {
                err = -EMSGSIZE;
                goto out;
        }

        if (len > SKB_MAX_ALLOC) {
                data_len = min_t(size_t,
                                 len - SKB_MAX_ALLOC,
                                 MAX_SKB_FRAGS * PAGE_SIZE);
                data_len = PAGE_ALIGN(data_len);

                BUILD_BUG_ON(SKB_MAX_ALLOC < PAGE_SIZE);
        }

        skb = sock_alloc_send_pskb(sk, len - data_len, data_len,
                                   msg->msg_flags & MSG_DONTWAIT, &err,
                                   PAGE_ALLOC_COSTLY_ORDER);
        if (!skb)
                goto out;

        err = unix_scm_to_skb(&scm, skb, true);
        if (err < 0)
                goto out_free;

        skb_put(skb, len - data_len);
        skb->data_len = data_len;
        skb->len = len;
        err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, len);
        if (err)
                goto out_free;

        timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);

        if (msg->msg_namelen) {
lookup:
                other = unix_find_other(sock_net(sk), msg->msg_name,
                                        msg->msg_namelen, sk->sk_type, 0);
                if (IS_ERR(other)) {
                        err = PTR_ERR(other);
                        goto out_free;
                }
        } else {
                other = unix_peer_get(sk);
                if (!other) {
                        err = -ENOTCONN;
                        goto out_free;
                }
        }

        if (sk_filter(other, skb) < 0) {
                /* Toss the packet but do not return any error to the sender */
                err = len;
                goto out_sock_put;
        }

        err = unix_maybe_add_creds(skb, sk, other);
        if (err)
                goto out_sock_put;

restart:
        sk_locked = 0;
        unix_state_lock(other);
restart_locked:

        if (!unix_may_send(sk, other)) {
                err = -EPERM;
                goto out_unlock;
        }

        if (unlikely(sock_flag(other, SOCK_DEAD))) {
                /* Check with 1003.1g - what should datagram error */

                unix_state_unlock(other);

                if (sk->sk_type == SOCK_SEQPACKET) {
                        /* We are here only when racing with unix_release_sock()
                         * is clearing @other. Never change state to TCP_CLOSE
                         * unlike SOCK_DGRAM wants.
                         */
                        err = -EPIPE;
                        goto out_sock_put;
                }

                if (!sk_locked)
                        unix_state_lock(sk);

                if (unix_peer(sk) == other) {
                        unix_peer(sk) = NULL;
                        unix_dgram_peer_wake_disconnect_wakeup(sk, other);

                        WRITE_ONCE(sk->sk_state, TCP_CLOSE);
                        unix_state_unlock(sk);

                        unix_dgram_disconnected(sk, other);
                        sock_put(other);
                        err = -ECONNREFUSED;
                        goto out_sock_put;
                }

                unix_state_unlock(sk);

                if (!msg->msg_namelen) {
                        err = -ECONNRESET;
                        goto out_sock_put;
                }

                sock_put(other);
                goto lookup;
        }

        if (other->sk_shutdown & RCV_SHUTDOWN) {
                err = -EPIPE;
                goto out_unlock;
        }

        if (UNIXCB(skb).fp && !other->sk_scm_rights) {
                err = -EPERM;
                goto out_unlock;
        }

        if (sk->sk_type != SOCK_SEQPACKET) {
                err = security_unix_may_send(sk->sk_socket, other->sk_socket);
                if (err)
                        goto out_unlock;
        }

        /* other == sk && unix_peer(other) != sk if
         * - unix_peer(sk) == NULL, destination address bound to sk
         * - unix_peer(sk) == sk by time of get but disconnected before lock
         */
        if (other != sk &&
            unlikely(unix_peer(other) != sk &&
            unix_recvq_full_lockless(other))) {
                if (timeo) {
                        timeo = unix_wait_for_peer(other, timeo);

                        err = sock_intr_errno(timeo);
                        if (signal_pending(current))
                                goto out_sock_put;

                        goto restart;
                }

                if (!sk_locked) {
                        unix_state_unlock(other);
                        unix_state_double_lock(sk, other);
                }

                if (unix_peer(sk) != other ||
                    unix_dgram_peer_wake_me(sk, other)) {
                        err = -EAGAIN;
                        sk_locked = 1;
                        goto out_unlock;
                }

                if (!sk_locked) {
                        sk_locked = 1;
                        goto restart_locked;
                }
        }

        if (unlikely(sk_locked))
                unix_state_unlock(sk);

        if (sock_flag(other, SOCK_RCVTSTAMP))
                __net_timestamp(skb);

        scm_stat_add(other, skb);
        skb_queue_tail(&other->sk_receive_queue, skb);
        unix_state_unlock(other);
        other->sk_data_ready(other);
        sock_put(other);
        scm_destroy(&scm);
        return len;

out_unlock:
        if (sk_locked)
                unix_state_unlock(sk);
        unix_state_unlock(other);
out_sock_put:
        sock_put(other);
out_free:
        consume_skb(skb);
out:
        scm_destroy(&scm);
        return err;
}

/* We use paged skbs for stream sockets, and limit occupancy to 32768
 * bytes, and a minimum of a full page.
 */
#define UNIX_SKB_FRAGS_SZ (PAGE_SIZE << get_order(32768))

#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
static int queue_oob(struct sock *sk, struct msghdr *msg, struct sock *other,
                     struct scm_cookie *scm, bool fds_sent)
{
        struct unix_sock *ousk = unix_sk(other);
        struct sk_buff *skb;
        int err;

        skb = sock_alloc_send_skb(sk, 1, msg->msg_flags & MSG_DONTWAIT, &err);

        if (!skb)
                return err;

        err = unix_scm_to_skb(scm, skb, !fds_sent);
        if (err < 0)
                goto out;

        err = unix_maybe_add_creds(skb, sk, other);
        if (err)
                goto out;

        skb_put(skb, 1);
        err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, 1);

        if (err)
                goto out;

        unix_state_lock(other);

        if (sock_flag(other, SOCK_DEAD) ||
            (other->sk_shutdown & RCV_SHUTDOWN)) {
                err = -EPIPE;
                goto out_unlock;
        }

        if (UNIXCB(skb).fp && !other->sk_scm_rights) {
                err = -EPERM;
                goto out_unlock;
        }

        scm_stat_add(other, skb);

        spin_lock(&other->sk_receive_queue.lock);
        WRITE_ONCE(ousk->oob_skb, skb);
        WRITE_ONCE(ousk->inq_len, ousk->inq_len + 1);
        __skb_queue_tail(&other->sk_receive_queue, skb);
        spin_unlock(&other->sk_receive_queue.lock);

        sk_send_sigurg(other);
        unix_state_unlock(other);
        other->sk_data_ready(other);

        return 0;
out_unlock:
        unix_state_unlock(other);
out:
        consume_skb(skb);
        return err;
}
#endif

static int unix_stream_sendmsg(struct socket *sock, struct msghdr *msg,
                               size_t len)
{
        struct sock *sk = sock->sk;
        struct sk_buff *skb = NULL;
        struct sock *other = NULL;
        struct unix_sock *otheru;
        struct scm_cookie scm;
        bool fds_sent = false;
        int err, sent = 0;

        err = scm_send(sock, msg, &scm, false);
        if (err < 0)
                return err;

        wait_for_unix_gc(scm.fp);

        if (msg->msg_flags & MSG_OOB) {
                err = -EOPNOTSUPP;
#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
                if (len)
                        len--;
                else
#endif
                        goto out_err;
        }

        if (msg->msg_namelen) {
                err = READ_ONCE(sk->sk_state) == TCP_ESTABLISHED ? -EISCONN : -EOPNOTSUPP;
                goto out_err;
        }

        other = unix_peer(sk);
        if (!other) {
                err = -ENOTCONN;
                goto out_err;
        }

        otheru = unix_sk(other);

        if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN)
                goto out_pipe;

        while (sent < len) {
                int size = len - sent;
                int data_len;

                if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES)) {
                        skb = sock_alloc_send_pskb(sk, 0, 0,
                                                   msg->msg_flags & MSG_DONTWAIT,
                                                   &err, 0);
                } else {
                        /* Keep two messages in the pipe so it schedules better */
                        size = min_t(int, size, (READ_ONCE(sk->sk_sndbuf) >> 1) - 64);

                        /* allow fallback to order-0 allocations */
                        size = min_t(int, size, SKB_MAX_HEAD(0) + UNIX_SKB_FRAGS_SZ);

                        data_len = max_t(int, 0, size - SKB_MAX_HEAD(0));

                        data_len = min_t(size_t, size, PAGE_ALIGN(data_len));

                        skb = sock_alloc_send_pskb(sk, size - data_len, data_len,
                                                   msg->msg_flags & MSG_DONTWAIT, &err,
                                                   get_order(UNIX_SKB_FRAGS_SZ));
                }
                if (!skb)
                        goto out_err;

                /* Only send the fds in the first buffer */
                err = unix_scm_to_skb(&scm, skb, !fds_sent);
                if (err < 0)
                        goto out_free;

                fds_sent = true;

                err = unix_maybe_add_creds(skb, sk, other);
                if (err)
                        goto out_free;

                if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES)) {
                        skb->ip_summed = CHECKSUM_UNNECESSARY;
                        err = skb_splice_from_iter(skb, &msg->msg_iter, size);
                        if (err < 0)
                                goto out_free;

                        size = err;
                        refcount_add(size, &sk->sk_wmem_alloc);
                } else {
                        skb_put(skb, size - data_len);
                        skb->data_len = data_len;
                        skb->len = size;
                        err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, size);
                        if (err)
                                goto out_free;
                }

                unix_state_lock(other);

                if (sock_flag(other, SOCK_DEAD) ||
                    (other->sk_shutdown & RCV_SHUTDOWN))
                        goto out_pipe_unlock;

                if (UNIXCB(skb).fp && !other->sk_scm_rights) {
                        unix_state_unlock(other);
                        err = -EPERM;
                        goto out_free;
                }

                scm_stat_add(other, skb);

                spin_lock(&other->sk_receive_queue.lock);
                WRITE_ONCE(otheru->inq_len, otheru->inq_len + skb->len);
                __skb_queue_tail(&other->sk_receive_queue, skb);
                spin_unlock(&other->sk_receive_queue.lock);

                unix_state_unlock(other);
                other->sk_data_ready(other);
                sent += size;
        }

#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
        if (msg->msg_flags & MSG_OOB) {
                err = queue_oob(sk, msg, other, &scm, fds_sent);
                if (err)
                        goto out_err;
                sent++;
        }
#endif

        scm_destroy(&scm);

        return sent;

out_pipe_unlock:
        unix_state_unlock(other);
out_pipe:
        if (!sent && !(msg->msg_flags & MSG_NOSIGNAL))
                send_sig(SIGPIPE, current, 0);
        err = -EPIPE;
out_free:
        consume_skb(skb);
out_err:
        scm_destroy(&scm);
        return sent ? : err;
}

static int unix_seqpacket_sendmsg(struct socket *sock, struct msghdr *msg,
                                  size_t len)
{
        int err;
        struct sock *sk = sock->sk;

        err = sock_error(sk);
        if (err)
                return err;

        if (READ_ONCE(sk->sk_state) != TCP_ESTABLISHED)
                return -ENOTCONN;

        if (msg->msg_namelen)
                msg->msg_namelen = 0;

        return unix_dgram_sendmsg(sock, msg, len);
}

static int unix_seqpacket_recvmsg(struct socket *sock, struct msghdr *msg,
                                  size_t size, int flags)
{
        struct sock *sk = sock->sk;

        if (READ_ONCE(sk->sk_state) != TCP_ESTABLISHED)
                return -ENOTCONN;

        return unix_dgram_recvmsg(sock, msg, size, flags);
}

static void unix_copy_addr(struct msghdr *msg, struct sock *sk)
{
        struct unix_address *addr = smp_load_acquire(&unix_sk(sk)->addr);

        if (addr) {
                msg->msg_namelen = addr->len;
                memcpy(msg->msg_name, addr->name, addr->len);
        }
}

int __unix_dgram_recvmsg(struct sock *sk, struct msghdr *msg, size_t size,
                         int flags)
{
        struct scm_cookie scm;
        struct socket *sock = sk->sk_socket;
        struct unix_sock *u = unix_sk(sk);
        struct sk_buff *skb, *last;
        long timeo;
        int skip;
        int err;

        err = -EOPNOTSUPP;
        if (flags&MSG_OOB)
                goto out;

        timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);

        do {
                mutex_lock(&u->iolock);

                skip = sk_peek_offset(sk, flags);
                skb = __skb_try_recv_datagram(sk, &sk->sk_receive_queue, flags,
                                              &skip, &err, &last);
                if (skb) {
                        if (!(flags & MSG_PEEK))
                                scm_stat_del(sk, skb);
                        break;
                }

                mutex_unlock(&u->iolock);

                if (err != -EAGAIN)
                        break;
        } while (timeo &&
                 !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue,
                                              &err, &timeo, last));

        if (!skb) { /* implies iolock unlocked */
                /* Signal EOF on disconnected non-blocking SEQPACKET socket. */
                if (sk->sk_type == SOCK_SEQPACKET && err == -EAGAIN &&
                    (READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN))
                        err = 0;
                goto out;
        }

        if (wq_has_sleeper(&u->peer_wait))
                wake_up_interruptible_sync_poll(&u->peer_wait,
                                                EPOLLOUT | EPOLLWRNORM |
                                                EPOLLWRBAND);

        if (msg->msg_name) {
                unix_copy_addr(msg, skb->sk);

                BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk,
                                                      msg->msg_name,
                                                      &msg->msg_namelen);
        }

        if (size > skb->len - skip)
                size = skb->len - skip;
        else if (size < skb->len - skip)
                msg->msg_flags |= MSG_TRUNC;

        err = skb_copy_datagram_msg(skb, skip, msg, size);
        if (err)
                goto out_free;

        if (sock_flag(sk, SOCK_RCVTSTAMP))
                __sock_recv_timestamp(msg, sk, skb);

        memset(&scm, 0, sizeof(scm));

        unix_skb_to_scm(skb, &scm);

        if (!(flags & MSG_PEEK)) {
                if (UNIXCB(skb).fp)
                        unix_detach_fds(&scm, skb);

                sk_peek_offset_bwd(sk, skb->len);
        } else {
                /* It is questionable: on PEEK we could:
                   - do not return fds - good, but too simple 8)
                   - return fds, and do not return them on read (old strategy,
                     apparently wrong)
                   - clone fds (I chose it for now, it is the most universal
                     solution)

                   POSIX 1003.1g does not actually define this clearly
                   at all. POSIX 1003.1g doesn't define a lot of things
                   clearly however!

                */

                sk_peek_offset_fwd(sk, size);

                if (UNIXCB(skb).fp)
                        unix_peek_fds(&scm, skb);
        }
        err = (flags & MSG_TRUNC) ? skb->len - skip : size;

        scm_recv_unix(sock, msg, &scm, flags);

out_free:
        skb_free_datagram(sk, skb);
        mutex_unlock(&u->iolock);
out:
        return err;
}

static int unix_dgram_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
                              int flags)
{
        struct sock *sk = sock->sk;

#ifdef CONFIG_BPF_SYSCALL
        const struct proto *prot = READ_ONCE(sk->sk_prot);

        if (prot != &unix_dgram_proto)
                return prot->recvmsg(sk, msg, size, flags, NULL);
#endif
        return __unix_dgram_recvmsg(sk, msg, size, flags);
}

static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
{
        struct unix_sock *u = unix_sk(sk);
        struct sk_buff *skb;
        int err;

        mutex_lock(&u->iolock);
        skb = skb_recv_datagram(sk, MSG_DONTWAIT, &err);
        mutex_unlock(&u->iolock);
        if (!skb)
                return err;

        return recv_actor(sk, skb);
}

/*
 *        Sleep until more data has arrived. But check for races..
 */
static long unix_stream_data_wait(struct sock *sk, long timeo,
                                  struct sk_buff *last, unsigned int last_len,
                                  bool freezable)
{
        unsigned int state = TASK_INTERRUPTIBLE | freezable * TASK_FREEZABLE;
        struct sk_buff *tail;
        DEFINE_WAIT(wait);

        unix_state_lock(sk);

        for (;;) {
                prepare_to_wait(sk_sleep(sk), &wait, state);

                tail = skb_peek_tail(&sk->sk_receive_queue);
                if (tail != last ||
                    (tail && tail->len != last_len) ||
                    sk->sk_err ||
                    (sk->sk_shutdown & RCV_SHUTDOWN) ||
                    signal_pending(current) ||
                    !timeo)
                        break;

                sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
                unix_state_unlock(sk);
                timeo = schedule_timeout(timeo);
                unix_state_lock(sk);

                if (sock_flag(sk, SOCK_DEAD))
                        break;

                sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
        }

        finish_wait(sk_sleep(sk), &wait);
        unix_state_unlock(sk);
        return timeo;
}

struct unix_stream_read_state {
        int (*recv_actor)(struct sk_buff *, int, int,
                          struct unix_stream_read_state *);
        struct socket *socket;
        struct msghdr *msg;
        struct pipe_inode_info *pipe;
        size_t size;
        int flags;
        unsigned int splice_flags;
};

#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
static int unix_stream_recv_urg(struct unix_stream_read_state *state)
{
        struct sk_buff *oob_skb, *read_skb = NULL;
        struct socket *sock = state->socket;
        struct sock *sk = sock->sk;
        struct unix_sock *u = unix_sk(sk);
        int chunk = 1;

        mutex_lock(&u->iolock);
        unix_state_lock(sk);
        spin_lock(&sk->sk_receive_queue.lock);

        if (sock_flag(sk, SOCK_URGINLINE) || !u->oob_skb) {
                spin_unlock(&sk->sk_receive_queue.lock);
                unix_state_unlock(sk);
                mutex_unlock(&u->iolock);
                return -EINVAL;
        }

        oob_skb = u->oob_skb;

        if (!(state->flags & MSG_PEEK)) {
                WRITE_ONCE(u->oob_skb, NULL);
                WRITE_ONCE(u->inq_len, u->inq_len - 1);

                if (oob_skb->prev != (struct sk_buff *)&sk->sk_receive_queue &&
                    !unix_skb_len(oob_skb->prev)) {
                        read_skb = oob_skb->prev;
                        __skb_unlink(read_skb, &sk->sk_receive_queue);
                }
        }

        spin_unlock(&sk->sk_receive_queue.lock);
        unix_state_unlock(sk);

        chunk = state->recv_actor(oob_skb, 0, chunk, state);

        if (!(state->flags & MSG_PEEK))
                UNIXCB(oob_skb).consumed += 1;

        mutex_unlock(&u->iolock);

        consume_skb(read_skb);

        if (chunk < 0)
                return -EFAULT;

        state->msg->msg_flags |= MSG_OOB;
        return 1;
}

static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk,
                                  int flags, int copied)
{
        struct sk_buff *read_skb = NULL, *unread_skb = NULL;
        struct unix_sock *u = unix_sk(sk);

        if (likely(unix_skb_len(skb) && skb != READ_ONCE(u->oob_skb)))
                return skb;

        spin_lock(&sk->sk_receive_queue.lock);

        if (!unix_skb_len(skb)) {
                if (copied && (!u->oob_skb || skb == u->oob_skb)) {
                        skb = NULL;
                } else if (flags & MSG_PEEK) {
                        skb = skb_peek_next(skb, &sk->sk_receive_queue);
                } else {
                        read_skb = skb;
                        skb = skb_peek_next(skb, &sk->sk_receive_queue);
                        __skb_unlink(read_skb, &sk->sk_receive_queue);
                }

                if (!skb)
                        goto unlock;
        }

        if (skb != u->oob_skb)
                goto unlock;

        if (copied) {
                skb = NULL;
        } else if (!(flags & MSG_PEEK)) {
                WRITE_ONCE(u->oob_skb, NULL);

                if (!sock_flag(sk, SOCK_URGINLINE)) {
                        __skb_unlink(skb, &sk->sk_receive_queue);
                        unread_skb = skb;
                        skb = skb_peek(&sk->sk_receive_queue);
                }
        } else if (!sock_flag(sk, SOCK_URGINLINE)) {
                skb = skb_peek_next(skb, &sk->sk_receive_queue);
        }

unlock:
        spin_unlock(&sk->sk_receive_queue.lock);

        consume_skb(read_skb);
        kfree_skb_reason(unread_skb, SKB_DROP_REASON_UNIX_SKIP_OOB);

        return skb;
}
#endif

static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
{
        struct sk_buff_head *queue = &sk->sk_receive_queue;
        struct unix_sock *u = unix_sk(sk);
        struct sk_buff *skb;
        int err;

        if (unlikely(READ_ONCE(sk->sk_state) != TCP_ESTABLISHED))
                return -ENOTCONN;

        err = sock_error(sk);
        if (err)
                return err;

        mutex_lock(&u->iolock);
        spin_lock(&queue->lock);

        skb = __skb_dequeue(queue);
        if (!skb) {
                spin_unlock(&queue->lock);
                mutex_unlock(&u->iolock);
                return -EAGAIN;
        }

        WRITE_ONCE(u->inq_len, u->inq_len - skb->len);

#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
        if (skb == u->oob_skb) {
                WRITE_ONCE(u->oob_skb, NULL);
                spin_unlock(&queue->lock);
                mutex_unlock(&u->iolock);

                kfree_skb_reason(skb, SKB_DROP_REASON_UNIX_SKIP_OOB);
                return -EAGAIN;
        }
#endif

        spin_unlock(&queue->lock);
        mutex_unlock(&u->iolock);

        return recv_actor(sk, skb);
}

static int unix_stream_read_generic(struct unix_stream_read_state *state,
                                    bool freezable)
{
        int noblock = state->flags & MSG_DONTWAIT;
        struct socket *sock = state->socket;
        struct msghdr *msg = state->msg;
        struct sock *sk = sock->sk;
        size_t size = state->size;
        int flags = state->flags;
        bool check_creds = false;
        struct scm_cookie scm;
        unsigned int last_len;
        struct unix_sock *u;
        int copied = 0;
        int err = 0;
        long timeo;
        int target;
        int skip;

        if (unlikely(READ_ONCE(sk->sk_state) != TCP_ESTABLISHED)) {
                err = -EINVAL;
                goto out;
        }

        if (unlikely(flags & MSG_OOB)) {
                err = -EOPNOTSUPP;
#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
                err = unix_stream_recv_urg(state);
#endif
                goto out;
        }

        target = sock_rcvlowat(sk, flags & MSG_WAITALL, size);
        timeo = sock_rcvtimeo(sk, noblock);

        memset(&scm, 0, sizeof(scm));

        u = unix_sk(sk);

        /* Lock the socket to prevent queue disordering
         * while sleeps in memcpy_tomsg
         */
        mutex_lock(&u->iolock);

        skip = max(sk_peek_offset(sk, flags), 0);

        do {
                struct sk_buff *skb, *last;
                int chunk;

redo:
                unix_state_lock(sk);
                if (sock_flag(sk, SOCK_DEAD)) {
                        err = -ECONNRESET;
                        goto unlock;
                }
                last = skb = skb_peek(&sk->sk_receive_queue);
                last_len = last ? last->len : 0;

again:
#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
                if (skb) {
                        skb = manage_oob(skb, sk, flags, copied);
                        if (!skb && copied) {
                                unix_state_unlock(sk);
                                break;
                        }
                }
#endif
                if (skb == NULL) {
                        if (copied >= target)
                                goto unlock;

                        /*
                         *        POSIX 1003.1g mandates this order.
                         */

                        err = sock_error(sk);
                        if (err)
                                goto unlock;
                        if (sk->sk_shutdown & RCV_SHUTDOWN)
                                goto unlock;

                        unix_state_unlock(sk);
                        if (!timeo) {
                                err = -EAGAIN;
                                break;
                        }

                        mutex_unlock(&u->iolock);

                        timeo = unix_stream_data_wait(sk, timeo, last,
                                                      last_len, freezable);

                        if (signal_pending(current)) {
                                err = sock_intr_errno(timeo);
                                scm_destroy(&scm);
                                goto out;
                        }

                        mutex_lock(&u->iolock);
                        goto redo;
unlock:
                        unix_state_unlock(sk);
                        break;
                }

                while (skip >= unix_skb_len(skb)) {
                        skip -= unix_skb_len(skb);
                        last = skb;
                        last_len = skb->len;
                        skb = skb_peek_next(skb, &sk->sk_receive_queue);
                        if (!skb)
                                goto again;
                }

                unix_state_unlock(sk);

                if (check_creds) {
                        /* Never glue messages from different writers */
                        if (!unix_skb_scm_eq(skb, &scm))
                                break;
                } else if (unix_may_passcred(sk)) {
                        /* Copy credentials */
                        unix_skb_to_scm(skb, &scm);
                        check_creds = true;
                }

                /* Copy address just once */
                if (msg && msg->msg_name) {
                        DECLARE_SOCKADDR(struct sockaddr_un *, sunaddr, msg->msg_name);

                        unix_copy_addr(msg, skb->sk);
                        BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, msg->msg_name,
                                                              &msg->msg_namelen);

                        sunaddr = NULL;
                }

                chunk = min_t(unsigned int, unix_skb_len(skb) - skip, size);
                chunk = state->recv_actor(skb, skip, chunk, state);
                if (chunk < 0) {
                        if (copied == 0)
                                copied = -EFAULT;
                        break;
                }
                copied += chunk;
                size -= chunk;

                /* Mark read part of skb as used */
                if (!(flags & MSG_PEEK)) {
                        UNIXCB(skb).consumed += chunk;

                        sk_peek_offset_bwd(sk, chunk);

                        if (UNIXCB(skb).fp) {
                                scm_stat_del(sk, skb);
                                unix_detach_fds(&scm, skb);
                        }

                        if (unix_skb_len(skb))
                                break;

                        spin_lock(&sk->sk_receive_queue.lock);
                        WRITE_ONCE(u->inq_len, u->inq_len - skb->len);
                        __skb_unlink(skb, &sk->sk_receive_queue);
                        spin_unlock(&sk->sk_receive_queue.lock);

                        consume_skb(skb);

                        if (scm.fp)
                                break;
                } else {
                        /* It is questionable, see note in unix_dgram_recvmsg.
                         */
                        if (UNIXCB(skb).fp)
                                unix_peek_fds(&scm, skb);

                        sk_peek_offset_fwd(sk, chunk);

                        if (UNIXCB(skb).fp)
                                break;

                        skip = 0;
                        last = skb;
                        last_len = skb->len;
                        unix_state_lock(sk);
                        skb = skb_peek_next(skb, &sk->sk_receive_queue);
                        if (skb)
                                goto again;
                        unix_state_unlock(sk);
                        break;
                }
        } while (size);

        mutex_unlock(&u->iolock);
        if (msg) {
                scm_recv_unix(sock, msg, &scm, flags);

                if (READ_ONCE(u->recvmsg_inq) || msg->msg_get_inq) {
                        msg->msg_inq = READ_ONCE(u->inq_len);
                        put_cmsg(msg, SOL_SOCKET, SCM_INQ,
                                 sizeof(msg->msg_inq), &msg->msg_inq);
                }
        } else {
                scm_destroy(&scm);
        }
out:
        return copied ? : err;
}

static int unix_stream_read_actor(struct sk_buff *skb,
                                  int skip, int chunk,
                                  struct unix_stream_read_state *state)
{
        int ret;

        ret = skb_copy_datagram_msg(skb, UNIXCB(skb).consumed + skip,
                                    state->msg, chunk);
        return ret ?: chunk;
}

int __unix_stream_recvmsg(struct sock *sk, struct msghdr *msg,
                          size_t size, int flags)
{
        struct unix_stream_read_state state = {
                .recv_actor = unix_stream_read_actor,
                .socket = sk->sk_socket,
                .msg = msg,
                .size = size,
                .flags = flags
        };

        return unix_stream_read_generic(&state, true);
}

static int unix_stream_recvmsg(struct socket *sock, struct msghdr *msg,
                               size_t size, int flags)
{
        struct unix_stream_read_state state = {
                .recv_actor = unix_stream_read_actor,
                .socket = sock,
                .msg = msg,
                .size = size,
                .flags = flags
        };

#ifdef CONFIG_BPF_SYSCALL
        struct sock *sk = sock->sk;
        const struct proto *prot = READ_ONCE(sk->sk_prot);

        if (prot != &unix_stream_proto)
                return prot->recvmsg(sk, msg, size, flags, NULL);
#endif
        return unix_stream_read_generic(&state, true);
}

static int unix_stream_splice_actor(struct sk_buff *skb,
                                    int skip, int chunk,
                                    struct unix_stream_read_state *state)
{
        return skb_splice_bits(skb, state->socket->sk,
                               UNIXCB(skb).consumed + skip,
                               state->pipe, chunk, state->splice_flags);
}

static ssize_t unix_stream_splice_read(struct socket *sock,  loff_t *ppos,
                                       struct pipe_inode_info *pipe,
                                       size_t size, unsigned int flags)
{
        struct unix_stream_read_state state = {
                .recv_actor = unix_stream_splice_actor,
                .socket = sock,
                .pipe = pipe,
                .size = size,
                .splice_flags = flags,
        };

        if (unlikely(*ppos))
                return -ESPIPE;

        if (sock->file->f_flags & O_NONBLOCK ||
            flags & SPLICE_F_NONBLOCK)
                state.flags = MSG_DONTWAIT;

        return unix_stream_read_generic(&state, false);
}

static int unix_shutdown(struct socket *sock, int mode)
{
        struct sock *sk = sock->sk;
        struct sock *other;

        if (mode < SHUT_RD || mode > SHUT_RDWR)
                return -EINVAL;
        /* This maps:
         * SHUT_RD   (0) -> RCV_SHUTDOWN  (1)
         * SHUT_WR   (1) -> SEND_SHUTDOWN (2)
         * SHUT_RDWR (2) -> SHUTDOWN_MASK (3)
         */
        ++mode;

        unix_state_lock(sk);
        WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | mode);
        other = unix_peer(sk);
        if (other)
                sock_hold(other);
        unix_state_unlock(sk);
        sk->sk_state_change(sk);

        if (other &&
                (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET)) {

                int peer_mode = 0;
                const struct proto *prot = READ_ONCE(other->sk_prot);

                if (prot->unhash)
                        prot->unhash(other);
                if (mode&RCV_SHUTDOWN)
                        peer_mode |= SEND_SHUTDOWN;
                if (mode&SEND_SHUTDOWN)
                        peer_mode |= RCV_SHUTDOWN;
                unix_state_lock(other);
                WRITE_ONCE(other->sk_shutdown, other->sk_shutdown | peer_mode);
                unix_state_unlock(other);
                other->sk_state_change(other);
                if (peer_mode == SHUTDOWN_MASK)
                        sk_wake_async(other, SOCK_WAKE_WAITD, POLL_HUP);
                else if (peer_mode & RCV_SHUTDOWN)
                        sk_wake_async(other, SOCK_WAKE_WAITD, POLL_IN);
        }
        if (other)
                sock_put(other);

        return 0;
}

long unix_inq_len(struct sock *sk)
{
        struct sk_buff *skb;
        long amount = 0;

        if (READ_ONCE(sk->sk_state) == TCP_LISTEN)
                return -EINVAL;

        if (sk->sk_type == SOCK_STREAM)
                return READ_ONCE(unix_sk(sk)->inq_len);

        spin_lock(&sk->sk_receive_queue.lock);
        if (sk->sk_type == SOCK_SEQPACKET) {
                skb_queue_walk(&sk->sk_receive_queue, skb)
                        amount += unix_skb_len(skb);
        } else {
                skb = skb_peek(&sk->sk_receive_queue);
                if (skb)
                        amount = skb->len;
        }
        spin_unlock(&sk->sk_receive_queue.lock);

        return amount;
}
EXPORT_SYMBOL_GPL(unix_inq_len);

long unix_outq_len(struct sock *sk)
{
        return sk_wmem_alloc_get(sk);
}
EXPORT_SYMBOL_GPL(unix_outq_len);

static int unix_open_file(struct sock *sk)
{
        struct file *f;
        int fd;

        if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
                return -EPERM;

        if (!smp_load_acquire(&unix_sk(sk)->addr))
                return -ENOENT;

        if (!unix_sk(sk)->path.dentry)
                return -ENOENT;

        fd = get_unused_fd_flags(O_CLOEXEC);
        if (fd < 0)
                return fd;

        f = dentry_open(&unix_sk(sk)->path, O_PATH, current_cred());
        if (IS_ERR(f)) {
                put_unused_fd(fd);
                return PTR_ERR(f);
        }

        fd_install(fd, f);
        return fd;
}

static int unix_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
{
        struct sock *sk = sock->sk;
        long amount = 0;
        int err;

        switch (cmd) {
        case SIOCOUTQ:
                amount = unix_outq_len(sk);
                err = put_user(amount, (int __user *)arg);
                break;
        case SIOCINQ:
                amount = unix_inq_len(sk);
                if (amount < 0)
                        err = amount;
                else
                        err = put_user(amount, (int __user *)arg);
                break;
        case SIOCUNIXFILE:
                err = unix_open_file(sk);
                break;
#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
        case SIOCATMARK:
                {
                        struct unix_sock *u = unix_sk(sk);
                        struct sk_buff *skb;
                        int answ = 0;

                        mutex_lock(&u->iolock);

                        skb = skb_peek(&sk->sk_receive_queue);
                        if (skb) {
                                struct sk_buff *oob_skb = READ_ONCE(u->oob_skb);
                                struct sk_buff *next_skb;

                                next_skb = skb_peek_next(skb, &sk->sk_receive_queue);

                                if (skb == oob_skb ||
                                    (!unix_skb_len(skb) &&
                                     (!oob_skb || next_skb == oob_skb)))
                                        answ = 1;
                        }

                        mutex_unlock(&u->iolock);

                        err = put_user(answ, (int __user *)arg);
                }
                break;
#endif
        default:
                err = -ENOIOCTLCMD;
                break;
        }
        return err;
}

#ifdef CONFIG_COMPAT
static int unix_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
{
        return unix_ioctl(sock, cmd, (unsigned long)compat_ptr(arg));
}
#endif

static __poll_t unix_poll(struct file *file, struct socket *sock, poll_table *wait)
{
        struct sock *sk = sock->sk;
        unsigned char state;
        __poll_t mask;
        u8 shutdown;

        sock_poll_wait(file, sock, wait);
        mask = 0;
        shutdown = READ_ONCE(sk->sk_shutdown);
        state = READ_ONCE(sk->sk_state);

        /* exceptional events? */
        if (READ_ONCE(sk->sk_err))
                mask |= EPOLLERR;
        if (shutdown == SHUTDOWN_MASK)
                mask |= EPOLLHUP;
        if (shutdown & RCV_SHUTDOWN)
                mask |= EPOLLRDHUP | EPOLLIN | EPOLLRDNORM;

        /* readable? */
        if (!skb_queue_empty_lockless(&sk->sk_receive_queue))
                mask |= EPOLLIN | EPOLLRDNORM;
        if (sk_is_readable(sk))
                mask |= EPOLLIN | EPOLLRDNORM;
#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
        if (READ_ONCE(unix_sk(sk)->oob_skb))
                mask |= EPOLLPRI;
#endif

        /* Connection-based need to check for termination and startup */
        if ((sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) &&
            state == TCP_CLOSE)
                mask |= EPOLLHUP;

        /*
         * we set writable also when the other side has shut down the
         * connection. This prevents stuck sockets.
         */
        if (unix_writable(sk, state))
                mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;

        return mask;
}

static __poll_t unix_dgram_poll(struct file *file, struct socket *sock,
                                    poll_table *wait)
{
        struct sock *sk = sock->sk, *other;
        unsigned int writable;
        unsigned char state;
        __poll_t mask;
        u8 shutdown;

        sock_poll_wait(file, sock, wait);
        mask = 0;
        shutdown = READ_ONCE(sk->sk_shutdown);
        state = READ_ONCE(sk->sk_state);

        /* exceptional events? */
        if (READ_ONCE(sk->sk_err) ||
            !skb_queue_empty_lockless(&sk->sk_error_queue))
                mask |= EPOLLERR |
                        (sock_flag(sk, SOCK_SELECT_ERR_QUEUE) ? EPOLLPRI : 0);

        if (shutdown & RCV_SHUTDOWN)
                mask |= EPOLLRDHUP | EPOLLIN | EPOLLRDNORM;
        if (shutdown == SHUTDOWN_MASK)
                mask |= EPOLLHUP;

        /* readable? */
        if (!skb_queue_empty_lockless(&sk->sk_receive_queue))
                mask |= EPOLLIN | EPOLLRDNORM;
        if (sk_is_readable(sk))
                mask |= EPOLLIN | EPOLLRDNORM;

        /* Connection-based need to check for termination and startup */
        if (sk->sk_type == SOCK_SEQPACKET && state == TCP_CLOSE)
                mask |= EPOLLHUP;

        /* No write status requested, avoid expensive OUT tests. */
        if (!(poll_requested_events(wait) & (EPOLLWRBAND|EPOLLWRNORM|EPOLLOUT)))
                return mask;

        writable = unix_writable(sk, state);
        if (writable) {
                unix_state_lock(sk);

                other = unix_peer(sk);
                if (other && unix_peer(other) != sk &&
                    unix_recvq_full_lockless(other) &&
                    unix_dgram_peer_wake_me(sk, other))
                        writable = 0;

                unix_state_unlock(sk);
        }

        if (writable)
                mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;
        else
                sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);

        return mask;
}

#ifdef CONFIG_PROC_FS

#define BUCKET_SPACE (BITS_PER_LONG - (UNIX_HASH_BITS + 1) - 1)

#define get_bucket(x) ((x) >> BUCKET_SPACE)
#define get_offset(x) ((x) & ((1UL << BUCKET_SPACE) - 1))
#define set_bucket_offset(b, o) ((b) << BUCKET_SPACE | (o))

static struct sock *unix_from_bucket(struct seq_file *seq, loff_t *pos)
{
        unsigned long offset = get_offset(*pos);
        unsigned long bucket = get_bucket(*pos);
        unsigned long count = 0;
        struct sock *sk;

        for (sk = sk_head(&seq_file_net(seq)->unx.table.buckets[bucket]);
             sk; sk = sk_next(sk)) {
                if (++count == offset)
                        break;
        }

        return sk;
}

static struct sock *unix_get_first(struct seq_file *seq, loff_t *pos)
{
        unsigned long bucket = get_bucket(*pos);
        struct net *net = seq_file_net(seq);
        struct sock *sk;

        while (bucket < UNIX_HASH_SIZE) {
                spin_lock(&net->unx.table.locks[bucket]);

                sk = unix_from_bucket(seq, pos);
                if (sk)
                        return sk;

                spin_unlock(&net->unx.table.locks[bucket]);

                *pos = set_bucket_offset(++bucket, 1);
        }

        return NULL;
}

static struct sock *unix_get_next(struct seq_file *seq, struct sock *sk,
                                  loff_t *pos)
{
        unsigned long bucket = get_bucket(*pos);

        sk = sk_next(sk);
        if (sk)
                return sk;


        spin_unlock(&seq_file_net(seq)->unx.table.locks[bucket]);

        *pos = set_bucket_offset(++bucket, 1);

        return unix_get_first(seq, pos);
}

static void *unix_seq_start(struct seq_file *seq, loff_t *pos)
{
        if (!*pos)
                return SEQ_START_TOKEN;

        return unix_get_first(seq, pos);
}

static void *unix_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        ++*pos;

        if (v == SEQ_START_TOKEN)
                return unix_get_first(seq, pos);

        return unix_get_next(seq, v, pos);
}

static void unix_seq_stop(struct seq_file *seq, void *v)
{
        struct sock *sk = v;

        if (sk)
                spin_unlock(&seq_file_net(seq)->unx.table.locks[sk->sk_hash]);
}

static int unix_seq_show(struct seq_file *seq, void *v)
{

        if (v == SEQ_START_TOKEN)
                seq_puts(seq, "Num       RefCount Protocol Flags    Type St "
                         "Inode Path\n");
        else {
                struct sock *s = v;
                struct unix_sock *u = unix_sk(s);
                unix_state_lock(s);

                seq_printf(seq, "%pK: %08X %08X %08X %04X %02X %5lu",
                        s,
                        refcount_read(&s->sk_refcnt),
                        0,
                        s->sk_state == TCP_LISTEN ? __SO_ACCEPTCON : 0,
                        s->sk_type,
                        s->sk_socket ?
                        (s->sk_state == TCP_ESTABLISHED ? SS_CONNECTED : SS_UNCONNECTED) :
                        (s->sk_state == TCP_ESTABLISHED ? SS_CONNECTING : SS_DISCONNECTING),
                        sock_i_ino(s));

                if (u->addr) {        // under a hash table lock here
                        int i, len;
                        seq_putc(seq, ' ');

                        i = 0;
                        len = u->addr->len -
                                offsetof(struct sockaddr_un, sun_path);
                        if (u->addr->name->sun_path[0]) {
                                len--;
                        } else {
                                seq_putc(seq, '@');
                                i++;
                        }
                        for ( ; i < len; i++)
                                seq_putc(seq, u->addr->name->sun_path[i] ?:
                                         '@');
                }
                unix_state_unlock(s);
                seq_putc(seq, '\n');
        }

        return 0;
}

static const struct seq_operations unix_seq_ops = {
        .start  = unix_seq_start,
        .next   = unix_seq_next,
        .stop   = unix_seq_stop,
        .show   = unix_seq_show,
};

#ifdef CONFIG_BPF_SYSCALL
struct bpf_unix_iter_state {
        struct seq_net_private p;
        unsigned int cur_sk;
        unsigned int end_sk;
        unsigned int max_sk;
        struct sock **batch;
        bool st_bucket_done;
};

struct bpf_iter__unix {
        __bpf_md_ptr(struct bpf_iter_meta *, meta);
        __bpf_md_ptr(struct unix_sock *, unix_sk);
        uid_t uid __aligned(8);
};

static int unix_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
                              struct unix_sock *unix_sk, uid_t uid)
{
        struct bpf_iter__unix ctx;

        meta->seq_num--;  /* skip SEQ_START_TOKEN */
        ctx.meta = meta;
        ctx.unix_sk = unix_sk;
        ctx.uid = uid;
        return bpf_iter_run_prog(prog, &ctx);
}

static int bpf_iter_unix_hold_batch(struct seq_file *seq, struct sock *start_sk)

{
        struct bpf_unix_iter_state *iter = seq->private;
        unsigned int expected = 1;
        struct sock *sk;

        sock_hold(start_sk);
        iter->batch[iter->end_sk++] = start_sk;

        for (sk = sk_next(start_sk); sk; sk = sk_next(sk)) {
                if (iter->end_sk < iter->max_sk) {
                        sock_hold(sk);
                        iter->batch[iter->end_sk++] = sk;
                }

                expected++;
        }

        spin_unlock(&seq_file_net(seq)->unx.table.locks[start_sk->sk_hash]);

        return expected;
}

static void bpf_iter_unix_put_batch(struct bpf_unix_iter_state *iter)
{
        while (iter->cur_sk < iter->end_sk)
                sock_put(iter->batch[iter->cur_sk++]);
}

static int bpf_iter_unix_realloc_batch(struct bpf_unix_iter_state *iter,
                                       unsigned int new_batch_sz)
{
        struct sock **new_batch;

        new_batch = kvmalloc(sizeof(*new_batch) * new_batch_sz,
                             GFP_USER | __GFP_NOWARN);
        if (!new_batch)
                return -ENOMEM;

        bpf_iter_unix_put_batch(iter);
        kvfree(iter->batch);
        iter->batch = new_batch;
        iter->max_sk = new_batch_sz;

        return 0;
}

static struct sock *bpf_iter_unix_batch(struct seq_file *seq,
                                        loff_t *pos)
{
        struct bpf_unix_iter_state *iter = seq->private;
        unsigned int expected;
        bool resized = false;
        struct sock *sk;

        if (iter->st_bucket_done)
                *pos = set_bucket_offset(get_bucket(*pos) + 1, 1);

again:
        /* Get a new batch */
        iter->cur_sk = 0;
        iter->end_sk = 0;

        sk = unix_get_first(seq, pos);
        if (!sk)
                return NULL; /* Done */

        expected = bpf_iter_unix_hold_batch(seq, sk);

        if (iter->end_sk == expected) {
                iter->st_bucket_done = true;
                return sk;
        }

        if (!resized && !bpf_iter_unix_realloc_batch(iter, expected * 3 / 2)) {
                resized = true;
                goto again;
        }

        return sk;
}

static void *bpf_iter_unix_seq_start(struct seq_file *seq, loff_t *pos)
{
        if (!*pos)
                return SEQ_START_TOKEN;

        /* bpf iter does not support lseek, so it always
         * continue from where it was stop()-ped.
         */
        return bpf_iter_unix_batch(seq, pos);
}

static void *bpf_iter_unix_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct bpf_unix_iter_state *iter = seq->private;
        struct sock *sk;

        /* Whenever seq_next() is called, the iter->cur_sk is
         * done with seq_show(), so advance to the next sk in
         * the batch.
         */
        if (iter->cur_sk < iter->end_sk)
                sock_put(iter->batch[iter->cur_sk++]);

        ++*pos;

        if (iter->cur_sk < iter->end_sk)
                sk = iter->batch[iter->cur_sk];
        else
                sk = bpf_iter_unix_batch(seq, pos);

        return sk;
}

static int bpf_iter_unix_seq_show(struct seq_file *seq, void *v)
{
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;
        struct sock *sk = v;
        uid_t uid;
        bool slow;
        int ret;

        if (v == SEQ_START_TOKEN)
                return 0;

        slow = lock_sock_fast(sk);

        if (unlikely(sk_unhashed(sk))) {
                ret = SEQ_SKIP;
                goto unlock;
        }

        uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk));
        meta.seq = seq;
        prog = bpf_iter_get_info(&meta, false);
        ret = unix_prog_seq_show(prog, &meta, v, uid);
unlock:
        unlock_sock_fast(sk, slow);
        return ret;
}

static void bpf_iter_unix_seq_stop(struct seq_file *seq, void *v)
{
        struct bpf_unix_iter_state *iter = seq->private;
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;

        if (!v) {
                meta.seq = seq;
                prog = bpf_iter_get_info(&meta, true);
                if (prog)
                        (void)unix_prog_seq_show(prog, &meta, v, 0);
        }

        if (iter->cur_sk < iter->end_sk)
                bpf_iter_unix_put_batch(iter);
}

static const struct seq_operations bpf_iter_unix_seq_ops = {
        .start        = bpf_iter_unix_seq_start,
        .next        = bpf_iter_unix_seq_next,
        .stop        = bpf_iter_unix_seq_stop,
        .show        = bpf_iter_unix_seq_show,
};
#endif
#endif

static const struct net_proto_family unix_family_ops = {
        .family = PF_UNIX,
        .create = unix_create,
        .owner        = THIS_MODULE,
};


static int __net_init unix_net_init(struct net *net)
{
        int i;

        net->unx.sysctl_max_dgram_qlen = 10;
        if (unix_sysctl_register(net))
                goto out;

#ifdef CONFIG_PROC_FS
        if (!proc_create_net("unix", 0, net->proc_net, &unix_seq_ops,
                             sizeof(struct seq_net_private)))
                goto err_sysctl;
#endif

        net->unx.table.locks = kvmalloc_array(UNIX_HASH_SIZE,
                                              sizeof(spinlock_t), GFP_KERNEL);
        if (!net->unx.table.locks)
                goto err_proc;

        net->unx.table.buckets = kvmalloc_array(UNIX_HASH_SIZE,
                                                sizeof(struct hlist_head),
                                                GFP_KERNEL);
        if (!net->unx.table.buckets)
                goto free_locks;

        for (i = 0; i < UNIX_HASH_SIZE; i++) {
                spin_lock_init(&net->unx.table.locks[i]);
                lock_set_cmp_fn(&net->unx.table.locks[i], unix_table_lock_cmp_fn, NULL);
                INIT_HLIST_HEAD(&net->unx.table.buckets[i]);
        }

        return 0;

free_locks:
        kvfree(net->unx.table.locks);
err_proc:
#ifdef CONFIG_PROC_FS
        remove_proc_entry("unix", net->proc_net);
err_sysctl:
#endif
        unix_sysctl_unregister(net);
out:
        return -ENOMEM;
}

static void __net_exit unix_net_exit(struct net *net)
{
        kvfree(net->unx.table.buckets);
        kvfree(net->unx.table.locks);
        unix_sysctl_unregister(net);
        remove_proc_entry("unix", net->proc_net);
}

static struct pernet_operations unix_net_ops = {
        .init = unix_net_init,
        .exit = unix_net_exit,
};

#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
DEFINE_BPF_ITER_FUNC(unix, struct bpf_iter_meta *meta,
                     struct unix_sock *unix_sk, uid_t uid)

#define INIT_BATCH_SZ 16

static int bpf_iter_init_unix(void *priv_data, struct bpf_iter_aux_info *aux)
{
        struct bpf_unix_iter_state *iter = priv_data;
        int err;

        err = bpf_iter_init_seq_net(priv_data, aux);
        if (err)
                return err;

        err = bpf_iter_unix_realloc_batch(iter, INIT_BATCH_SZ);
        if (err) {
                bpf_iter_fini_seq_net(priv_data);
                return err;
        }

        return 0;
}

static void bpf_iter_fini_unix(void *priv_data)
{
        struct bpf_unix_iter_state *iter = priv_data;

        bpf_iter_fini_seq_net(priv_data);
        kvfree(iter->batch);
}

static const struct bpf_iter_seq_info unix_seq_info = {
        .seq_ops                = &bpf_iter_unix_seq_ops,
        .init_seq_private        = bpf_iter_init_unix,
        .fini_seq_private        = bpf_iter_fini_unix,
        .seq_priv_size                = sizeof(struct bpf_unix_iter_state),
};

static const struct bpf_func_proto *
bpf_iter_unix_get_func_proto(enum bpf_func_id func_id,
                             const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_setsockopt:
                return &bpf_sk_setsockopt_proto;
        case BPF_FUNC_getsockopt:
                return &bpf_sk_getsockopt_proto;
        default:
                return NULL;
        }
}

static struct bpf_iter_reg unix_reg_info = {
        .target                        = "unix",
        .ctx_arg_info_size        = 1,
        .ctx_arg_info                = {
                { offsetof(struct bpf_iter__unix, unix_sk),
                  PTR_TO_BTF_ID_OR_NULL },
        },
        .get_func_proto         = bpf_iter_unix_get_func_proto,
        .seq_info                = &unix_seq_info,
};

static void __init bpf_iter_register(void)
{
        unix_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UNIX];
        if (bpf_iter_reg_target(&unix_reg_info))
                pr_warn("Warning: could not register bpf iterator unix\n");
}
#endif

static int __init af_unix_init(void)
{
        int i, rc = -1;

        BUILD_BUG_ON(sizeof(struct unix_skb_parms) > sizeof_field(struct sk_buff, cb));

        for (i = 0; i < UNIX_HASH_SIZE / 2; i++) {
                spin_lock_init(&bsd_socket_locks[i]);
                INIT_HLIST_HEAD(&bsd_socket_buckets[i]);
        }

        rc = proto_register(&unix_dgram_proto, 1);
        if (rc != 0) {
                pr_crit("%s: Cannot create unix_sock SLAB cache!\n", __func__);
                goto out;
        }

        rc = proto_register(&unix_stream_proto, 1);
        if (rc != 0) {
                pr_crit("%s: Cannot create unix_sock SLAB cache!\n", __func__);
                proto_unregister(&unix_dgram_proto);
                goto out;
        }

        sock_register(&unix_family_ops);
        register_pernet_subsys(&unix_net_ops);
        unix_bpf_build_proto();

#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
        bpf_iter_register();
#endif

out:
        return rc;
}

/* Later than subsys_initcall() because we depend on stuff initialised there */
fs_initcall(af_unix_init);





















   52 

   14 











   43 









   43 
   43 
   43 







    2 

    2 
    2 









    2 
    2 

    2 









    2 












    4 











    5 




    2 



    3 

















    1 











    3 


    3 




    3 













    2 



    2 







    2 















    2 




    2 
















    2 




    2 






    2 


































































    7 

    7 























   47 



    1 

   46 



   45 








    1 










    7 

























   47 
   47 

    2 














   47 



   47 














   47 




   47 


   47 












    4 


    4 

    4 
    4 








    2 

    2 




    2 







    2 












    2 












    4 











   35 



   46 
















    4 








    4 


















    4 
    4 
















































   12 

    2 





    9 








    1 















    1 












   19 





   20 

    4 


   16 
   16 



















   15 

   15 
    4 

   12 












   13 


   13 





   13 
















   11 



   11 





    1 
   10 

   11 














   11 



   10 











   11 

   10 






   41 
   41 

   42 


   42 







   42 
   42 

   42 

   42 

   42 




    3 






    3 

    3 







    2 





    3 









    3 





    2 


























   15 


   15 

   15 

   15 










   52 










   52 


   12 




   43 


   43 

   12 




   12 





   11 








   52 



    2 


   52 

   12 


   12 





   43 
   42 






    4 


    2 

   50 









   16 




   16 




   16 


   13 

    1 



    1 











    3 







   14 























   16 
   14 

    2 

   15 











   10 










   10 







   10 

   10 




   10 













   10 


   10 

















































    2 
























































    2 














   11 
























    2 





    2 

    2 
    2 


    2 

    2 






    2 

    2 



    2 













    1 




























   13 






    6 


    7 




















































































































































    1 



    1 


    1 










    1 













    1 






    1 
    1 







    1 




    1 
























































    1 




    1 

















    1 


    1 

















    1 





    1 











    1 

    1 



    1 



    1 






    1 













    1 


    1 







    1 





    1 



    1 








    1 
    1 



    1 









    1 


    1 







    1 











    1 











    1 










    1 

    1 

    1 


    1 
    1 



    1 










    1 































    1 
    1 






    1 
    1 























   52 







   52 

   52 

   12 




   43 








    2 
   41 




   41 







   13 








   12 

   13 









   13 






    1 



   11 

   12 

    1 

   11 















   11 

    1 

    1 









   12 









    1 






    1 


    1 
































    1 


    1 






























































































































   69 





   69 

   51 
   43 





























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
// SPDX-License-Identifier: GPL-2.0-only
/*
 * mac80211 - channel management
 * Copyright 2020 - 2025 Intel Corporation
 */

#include <linux/nl80211.h>
#include <linux/export.h>
#include <linux/rtnetlink.h>
#include <net/cfg80211.h>
#include "ieee80211_i.h"
#include "driver-ops.h"
#include "rate.h"

static int ieee80211_chanctx_num_assigned(struct ieee80211_local *local,
                                          struct ieee80211_chanctx *ctx)
{
        struct ieee80211_link_data *link;
        int num = 0;

        lockdep_assert_wiphy(local->hw.wiphy);

        list_for_each_entry(link, &ctx->assigned_links, assigned_chanctx_list)
                num++;

        return num;
}

static int ieee80211_chanctx_num_reserved(struct ieee80211_local *local,
                                          struct ieee80211_chanctx *ctx)
{
        struct ieee80211_link_data *link;
        int num = 0;

        lockdep_assert_wiphy(local->hw.wiphy);

        list_for_each_entry(link, &ctx->reserved_links, reserved_chanctx_list)
                num++;

        return num;
}

int ieee80211_chanctx_refcount(struct ieee80211_local *local,
                               struct ieee80211_chanctx *ctx)
{
        return ieee80211_chanctx_num_assigned(local, ctx) +
               ieee80211_chanctx_num_reserved(local, ctx);
}

static int ieee80211_num_chanctx(struct ieee80211_local *local, int radio_idx)
{
        struct ieee80211_chanctx *ctx;
        int num = 0;

        lockdep_assert_wiphy(local->hw.wiphy);

        list_for_each_entry(ctx, &local->chanctx_list, list) {
                if (radio_idx >= 0 && ctx->conf.radio_idx != radio_idx)
                        continue;
                num++;
        }

        return num;
}

static bool ieee80211_can_create_new_chanctx(struct ieee80211_local *local,
                                             int radio_idx)
{
        lockdep_assert_wiphy(local->hw.wiphy);

        return ieee80211_num_chanctx(local, radio_idx) <
               ieee80211_max_num_channels(local, radio_idx);
}

static struct ieee80211_chanctx *
ieee80211_link_get_chanctx(struct ieee80211_link_data *link)
{
        struct ieee80211_local *local __maybe_unused = link->sdata->local;
        struct ieee80211_chanctx_conf *conf;

        conf = rcu_dereference_protected(link->conf->chanctx_conf,
                                         lockdep_is_held(&local->hw.wiphy->mtx));
        if (!conf)
                return NULL;

        return container_of(conf, struct ieee80211_chanctx, conf);
}

bool ieee80211_chanreq_identical(const struct ieee80211_chan_req *a,
                                 const struct ieee80211_chan_req *b)
{
        if (!cfg80211_chandef_identical(&a->oper, &b->oper))
                return false;
        if (!a->ap.chan && !b->ap.chan)
                return true;
        return cfg80211_chandef_identical(&a->ap, &b->ap);
}

static const struct ieee80211_chan_req *
ieee80211_chanreq_compatible(const struct ieee80211_chan_req *a,
                             const struct ieee80211_chan_req *b,
                             struct ieee80211_chan_req *tmp)
{
        const struct cfg80211_chan_def *compat;

        if (a->ap.chan && b->ap.chan &&
            !cfg80211_chandef_identical(&a->ap, &b->ap))
                return NULL;

        compat = cfg80211_chandef_compatible(&a->oper, &b->oper);
        if (!compat)
                return NULL;

        /* Note: later code assumes this always fills & returns tmp if compat */
        tmp->oper = *compat;
        tmp->ap = a->ap.chan ? a->ap : b->ap;
        return tmp;
}

static const struct ieee80211_chan_req *
ieee80211_chanctx_compatible(struct ieee80211_chanctx *ctx,
                             const struct ieee80211_chan_req *req,
                             struct ieee80211_chan_req *tmp)
{
        const struct ieee80211_chan_req *ret;
        struct ieee80211_chan_req tmp2;

        *tmp = (struct ieee80211_chan_req){
                .oper = ctx->conf.def,
                .ap = ctx->conf.ap,
        };

        ret = ieee80211_chanreq_compatible(tmp, req, &tmp2);
        if (!ret)
                return NULL;
        *tmp = *ret;
        return tmp;
}

static const struct ieee80211_chan_req *
ieee80211_chanctx_reserved_chanreq(struct ieee80211_local *local,
                                   struct ieee80211_chanctx *ctx,
                                   const struct ieee80211_chan_req *req,
                                   struct ieee80211_chan_req *tmp)
{
        struct ieee80211_link_data *link;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (WARN_ON(!req))
                return NULL;

        list_for_each_entry(link, &ctx->reserved_links, reserved_chanctx_list) {
                req = ieee80211_chanreq_compatible(&link->reserved, req, tmp);
                if (!req)
                        break;
        }

        return req;
}

static const struct ieee80211_chan_req *
ieee80211_chanctx_non_reserved_chandef(struct ieee80211_local *local,
                                       struct ieee80211_chanctx *ctx,
                                       const struct ieee80211_chan_req *compat,
                                       struct ieee80211_chan_req *tmp)
{
        struct ieee80211_link_data *link;
        const struct ieee80211_chan_req *comp_def = compat;

        lockdep_assert_wiphy(local->hw.wiphy);

        list_for_each_entry(link, &ctx->assigned_links, assigned_chanctx_list) {
                struct ieee80211_bss_conf *link_conf = link->conf;

                if (link->reserved_chanctx)
                        continue;

                comp_def = ieee80211_chanreq_compatible(&link_conf->chanreq,
                                                        comp_def, tmp);
                if (!comp_def)
                        break;
        }

        return comp_def;
}

static bool
ieee80211_chanctx_can_reserve(struct ieee80211_local *local,
                              struct ieee80211_chanctx *ctx,
                              const struct ieee80211_chan_req *req)
{
        struct ieee80211_chan_req tmp;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!ieee80211_chanctx_reserved_chanreq(local, ctx, req, &tmp))
                return false;

        if (!ieee80211_chanctx_non_reserved_chandef(local, ctx, req, &tmp))
                return false;

        if (!list_empty(&ctx->reserved_links) &&
            ieee80211_chanctx_reserved_chanreq(local, ctx, req, &tmp))
                return true;

        return false;
}

static struct ieee80211_chanctx *
ieee80211_find_reservation_chanctx(struct ieee80211_local *local,
                                   const struct ieee80211_chan_req *chanreq,
                                   enum ieee80211_chanctx_mode mode)
{
        struct ieee80211_chanctx *ctx;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (mode == IEEE80211_CHANCTX_EXCLUSIVE)
                return NULL;

        list_for_each_entry(ctx, &local->chanctx_list, list) {
                if (ctx->replace_state == IEEE80211_CHANCTX_WILL_BE_REPLACED)
                        continue;

                if (ctx->mode == IEEE80211_CHANCTX_EXCLUSIVE)
                        continue;

                if (!ieee80211_chanctx_can_reserve(local, ctx, chanreq))
                        continue;

                return ctx;
        }

        return NULL;
}

static enum nl80211_chan_width ieee80211_get_sta_bw(struct sta_info *sta,
                                                    unsigned int link_id)
{
        enum ieee80211_sta_rx_bandwidth width;
        struct link_sta_info *link_sta;

        link_sta = wiphy_dereference(sta->local->hw.wiphy, sta->link[link_id]);

        /* no effect if this STA has no presence on this link */
        if (!link_sta)
                return NL80211_CHAN_WIDTH_20_NOHT;

        /*
         * We assume that TX/RX might be asymmetric (so e.g. VHT operating
         * mode notification changes what a STA wants to receive, but not
         * necessarily what it will transmit to us), and therefore use the
         * capabilities here. Calling it RX bandwidth capability is a bit
         * wrong though, since capabilities are in fact symmetric.
         */
        width = ieee80211_sta_cap_rx_bw(link_sta);

        switch (width) {
        case IEEE80211_STA_RX_BW_20:
                if (link_sta->pub->ht_cap.ht_supported)
                        return NL80211_CHAN_WIDTH_20;
                else
                        return NL80211_CHAN_WIDTH_20_NOHT;
        case IEEE80211_STA_RX_BW_40:
                return NL80211_CHAN_WIDTH_40;
        case IEEE80211_STA_RX_BW_80:
                return NL80211_CHAN_WIDTH_80;
        case IEEE80211_STA_RX_BW_160:
                /*
                 * This applied for both 160 and 80+80. since we use
                 * the returned value to consider degradation of
                 * ctx->conf.min_def, we have to make sure to take
                 * the bigger one (NL80211_CHAN_WIDTH_160).
                 * Otherwise we might try degrading even when not
                 * needed, as the max required sta_bw returned (80+80)
                 * might be smaller than the configured bw (160).
                 */
                return NL80211_CHAN_WIDTH_160;
        case IEEE80211_STA_RX_BW_320:
                return NL80211_CHAN_WIDTH_320;
        default:
                WARN_ON(1);
                return NL80211_CHAN_WIDTH_20;
        }
}

static enum nl80211_chan_width
ieee80211_get_max_required_bw(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        unsigned int link_id = link->link_id;
        enum nl80211_chan_width max_bw = NL80211_CHAN_WIDTH_20_NOHT;
        struct sta_info *sta;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        list_for_each_entry(sta, &sdata->local->sta_list, list) {
                if (sdata != sta->sdata &&
                    !(sta->sdata->bss && sta->sdata->bss == sdata->bss))
                        continue;

                max_bw = max(max_bw, ieee80211_get_sta_bw(sta, link_id));
        }

        return max_bw;
}

static enum nl80211_chan_width
ieee80211_get_chanctx_max_required_bw(struct ieee80211_local *local,
                                      struct ieee80211_chanctx *ctx,
                                      struct ieee80211_link_data *rsvd_for,
                                      bool check_reserved)
{
        struct ieee80211_sub_if_data *sdata;
        struct ieee80211_link_data *link;
        enum nl80211_chan_width max_bw = NL80211_CHAN_WIDTH_20_NOHT;

        if (WARN_ON(check_reserved && rsvd_for))
                return ctx->conf.def.width;

        for_each_sdata_link(local, link) {
                enum nl80211_chan_width width = NL80211_CHAN_WIDTH_20_NOHT;

                if (check_reserved) {
                        if (link->reserved_chanctx != ctx)
                                continue;
                } else if (link != rsvd_for &&
                           rcu_access_pointer(link->conf->chanctx_conf) != &ctx->conf)
                        continue;

                switch (link->sdata->vif.type) {
                case NL80211_IFTYPE_STATION:
                        if (!link->sdata->vif.cfg.assoc) {
                                /*
                                 * The AP's sta->bandwidth may not yet be set
                                 * at this point (pre-association), so simply
                                 * take the width from the chandef. We cannot
                                 * have TDLS peers yet (only after association).
                                 */
                                width = link->conf->chanreq.oper.width;
                                break;
                        }
                        /*
                         * otherwise just use min_def like in AP, depending on what
                         * we currently think the AP STA (and possibly TDLS peers)
                         * require(s)
                         */
                        fallthrough;
                case NL80211_IFTYPE_AP:
                case NL80211_IFTYPE_AP_VLAN:
                        width = ieee80211_get_max_required_bw(link);
                        break;
                case NL80211_IFTYPE_P2P_DEVICE:
                case NL80211_IFTYPE_NAN:
                        continue;
                case NL80211_IFTYPE_MONITOR:
                        WARN_ON_ONCE(!ieee80211_hw_check(&local->hw,
                                                         NO_VIRTUAL_MONITOR));
                        fallthrough;
                case NL80211_IFTYPE_ADHOC:
                case NL80211_IFTYPE_MESH_POINT:
                case NL80211_IFTYPE_OCB:
                        width = link->conf->chanreq.oper.width;
                        break;
                case NL80211_IFTYPE_WDS:
                case NL80211_IFTYPE_UNSPECIFIED:
                case NUM_NL80211_IFTYPES:
                case NL80211_IFTYPE_P2P_CLIENT:
                case NL80211_IFTYPE_P2P_GO:
                        WARN_ON_ONCE(1);
                }

                max_bw = max(max_bw, width);
        }

        /* use the configured bandwidth in case of monitor interface */
        sdata = wiphy_dereference(local->hw.wiphy, local->monitor_sdata);
        if (sdata &&
            rcu_access_pointer(sdata->vif.bss_conf.chanctx_conf) == &ctx->conf)
                max_bw = max(max_bw, ctx->conf.def.width);

        return max_bw;
}

/*
 * recalc the min required chan width of the channel context, which is
 * the max of min required widths of all the interfaces bound to this
 * channel context.
 */
static u32
_ieee80211_recalc_chanctx_min_def(struct ieee80211_local *local,
                                  struct ieee80211_chanctx *ctx,
                                  struct ieee80211_link_data *rsvd_for,
                                  bool check_reserved)
{
        enum nl80211_chan_width max_bw;
        struct cfg80211_chan_def min_def;

        lockdep_assert_wiphy(local->hw.wiphy);

        /* don't optimize non-20MHz based and radar_enabled confs */
        if (ctx->conf.def.width == NL80211_CHAN_WIDTH_5 ||
            ctx->conf.def.width == NL80211_CHAN_WIDTH_10 ||
            ctx->conf.def.width == NL80211_CHAN_WIDTH_1 ||
            ctx->conf.def.width == NL80211_CHAN_WIDTH_2 ||
            ctx->conf.def.width == NL80211_CHAN_WIDTH_4 ||
            ctx->conf.def.width == NL80211_CHAN_WIDTH_8 ||
            ctx->conf.def.width == NL80211_CHAN_WIDTH_16 ||
            ctx->conf.radar_enabled) {
                ctx->conf.min_def = ctx->conf.def;
                return 0;
        }

        max_bw = ieee80211_get_chanctx_max_required_bw(local, ctx, rsvd_for,
                                                       check_reserved);

        /* downgrade chandef up to max_bw */
        min_def = ctx->conf.def;
        while (min_def.width > max_bw)
                ieee80211_chandef_downgrade(&min_def, NULL);

        if (cfg80211_chandef_identical(&ctx->conf.min_def, &min_def))
                return 0;

        ctx->conf.min_def = min_def;
        if (!ctx->driver_present)
                return 0;

        return IEEE80211_CHANCTX_CHANGE_MIN_DEF;
}

static void ieee80211_chan_bw_change(struct ieee80211_local *local,
                                     struct ieee80211_chanctx *ctx,
                                     bool reserved, bool narrowed)
{
        struct sta_info *sta;
        struct ieee80211_supported_band *sband =
                local->hw.wiphy->bands[ctx->conf.def.chan->band];

        rcu_read_lock();
        list_for_each_entry_rcu(sta, &local->sta_list,
                                list) {
                struct ieee80211_sub_if_data *sdata = sta->sdata;
                enum ieee80211_sta_rx_bandwidth new_sta_bw;
                unsigned int link_id;

                if (!ieee80211_sdata_running(sta->sdata))
                        continue;

                for (link_id = 0; link_id < ARRAY_SIZE(sta->sdata->link); link_id++) {
                        struct ieee80211_link_data *link =
                                rcu_dereference(sdata->link[link_id]);
                        struct ieee80211_bss_conf *link_conf;
                        struct cfg80211_chan_def *new_chandef;
                        struct link_sta_info *link_sta;

                        if (!link)
                                continue;

                        link_conf = link->conf;

                        if (rcu_access_pointer(link_conf->chanctx_conf) != &ctx->conf)
                                continue;

                        link_sta = rcu_dereference(sta->link[link_id]);
                        if (!link_sta)
                                continue;

                        if (reserved)
                                new_chandef = &link->reserved.oper;
                        else
                                new_chandef = &link_conf->chanreq.oper;

                        new_sta_bw = _ieee80211_sta_cur_vht_bw(link_sta,
                                                               new_chandef);

                        /* nothing change */
                        if (new_sta_bw == link_sta->pub->bandwidth)
                                continue;

                        /* vif changed to narrow BW and narrow BW for station wasn't
                         * requested or vice versa */
                        if ((new_sta_bw < link_sta->pub->bandwidth) == !narrowed)
                                continue;

                        link_sta->pub->bandwidth = new_sta_bw;
                        rate_control_rate_update(local, sband, link_sta,
                                                 IEEE80211_RC_BW_CHANGED);
                }
        }
        rcu_read_unlock();
}

/*
 * recalc the min required chan width of the channel context, which is
 * the max of min required widths of all the interfaces bound to this
 * channel context.
 */
void ieee80211_recalc_chanctx_min_def(struct ieee80211_local *local,
                                      struct ieee80211_chanctx *ctx,
                                      struct ieee80211_link_data *rsvd_for,
                                      bool check_reserved)
{
        u32 changed = _ieee80211_recalc_chanctx_min_def(local, ctx, rsvd_for,
                                                        check_reserved);

        if (!changed)
                return;

        /* check is BW narrowed */
        ieee80211_chan_bw_change(local, ctx, false, true);

        drv_change_chanctx(local, ctx, changed);

        /* check is BW wider */
        ieee80211_chan_bw_change(local, ctx, false, false);
}

static void _ieee80211_change_chanctx(struct ieee80211_local *local,
                                      struct ieee80211_chanctx *ctx,
                                      struct ieee80211_chanctx *old_ctx,
                                      const struct ieee80211_chan_req *chanreq,
                                      struct ieee80211_link_data *rsvd_for)
{
        const struct cfg80211_chan_def *chandef = &chanreq->oper;
        struct ieee80211_chan_req ctx_req = {
                .oper = ctx->conf.def,
                .ap = ctx->conf.ap,
        };
        u32 changed = 0;

        /* expected to handle only 20/40/80/160/320 channel widths */
        switch (chandef->width) {
        case NL80211_CHAN_WIDTH_20_NOHT:
        case NL80211_CHAN_WIDTH_20:
        case NL80211_CHAN_WIDTH_40:
        case NL80211_CHAN_WIDTH_80:
        case NL80211_CHAN_WIDTH_80P80:
        case NL80211_CHAN_WIDTH_160:
        case NL80211_CHAN_WIDTH_320:
                break;
        default:
                WARN_ON(1);
        }

        /* Check maybe BW narrowed - we do this _before_ calling recalc_chanctx_min_def
         * due to maybe not returning from it, e.g in case new context was added
         * first time with all parameters up to date.
         */
        ieee80211_chan_bw_change(local, old_ctx, false, true);

        if (ieee80211_chanreq_identical(&ctx_req, chanreq)) {
                ieee80211_recalc_chanctx_min_def(local, ctx, rsvd_for, false);
                return;
        }

        WARN_ON(ieee80211_chanctx_refcount(local, ctx) > 1 &&
                !cfg80211_chandef_compatible(&ctx->conf.def, &chanreq->oper));

        ieee80211_remove_wbrf(local, &ctx->conf.def);

        if (!cfg80211_chandef_identical(&ctx->conf.def, &chanreq->oper)) {
                if (ctx->conf.def.width != chanreq->oper.width)
                        changed |= IEEE80211_CHANCTX_CHANGE_WIDTH;
                if (ctx->conf.def.punctured != chanreq->oper.punctured)
                        changed |= IEEE80211_CHANCTX_CHANGE_PUNCTURING;
        }
        if (!cfg80211_chandef_identical(&ctx->conf.ap, &chanreq->ap))
                changed |= IEEE80211_CHANCTX_CHANGE_AP;
        ctx->conf.def = *chandef;
        ctx->conf.ap = chanreq->ap;

        /* check if min chanctx also changed */
        changed |= _ieee80211_recalc_chanctx_min_def(local, ctx, rsvd_for, false);

        ieee80211_add_wbrf(local, &ctx->conf.def);

        drv_change_chanctx(local, ctx, changed);

        /* check if BW is wider */
        ieee80211_chan_bw_change(local, old_ctx, false, false);
}

static void ieee80211_change_chanctx(struct ieee80211_local *local,
                                     struct ieee80211_chanctx *ctx,
                                     struct ieee80211_chanctx *old_ctx,
                                     const struct ieee80211_chan_req *chanreq)
{
        _ieee80211_change_chanctx(local, ctx, old_ctx, chanreq, NULL);
}

/* Note: if successful, the returned chanctx is reserved for the link */
static struct ieee80211_chanctx *
ieee80211_find_chanctx(struct ieee80211_local *local,
                       struct ieee80211_link_data *link,
                       const struct ieee80211_chan_req *chanreq,
                       enum ieee80211_chanctx_mode mode)
{
        struct ieee80211_chan_req tmp;
        struct ieee80211_chanctx *ctx;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (mode == IEEE80211_CHANCTX_EXCLUSIVE)
                return NULL;

        if (WARN_ON(link->reserved_chanctx))
                return NULL;

        list_for_each_entry(ctx, &local->chanctx_list, list) {
                const struct ieee80211_chan_req *compat;

                if (ctx->replace_state != IEEE80211_CHANCTX_REPLACE_NONE)
                        continue;

                if (ctx->mode == IEEE80211_CHANCTX_EXCLUSIVE)
                        continue;

                compat = ieee80211_chanctx_compatible(ctx, chanreq, &tmp);
                if (!compat)
                        continue;

                compat = ieee80211_chanctx_reserved_chanreq(local, ctx,
                                                            compat, &tmp);
                if (!compat)
                        continue;

                /*
                 * Reserve the chanctx temporarily, as the driver might change
                 * active links during callbacks we make into it below and/or
                 * later during assignment, which could (otherwise) cause the
                 * context to actually be removed.
                 */
                link->reserved_chanctx = ctx;
                list_add(&link->reserved_chanctx_list,
                         &ctx->reserved_links);

                ieee80211_change_chanctx(local, ctx, ctx, compat);

                return ctx;
        }

        return NULL;
}

bool ieee80211_is_radar_required(struct ieee80211_local *local,
                                 struct cfg80211_scan_request *req)
{
        struct wiphy *wiphy = local->hw.wiphy;
        struct ieee80211_link_data *link;
        struct ieee80211_channel *chan;
        int radio_idx;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!req)
                return false;

        for_each_sdata_link(local, link) {
                if (link->radar_required) {
                        chan = link->conf->chanreq.oper.chan;
                        radio_idx = cfg80211_get_radio_idx_by_chan(wiphy, chan);

                        if (ieee80211_is_radio_idx_in_scan_req(wiphy, req,
                                                               radio_idx))
                                return true;
                }
        }

        return false;
}

static bool
ieee80211_chanctx_radar_required(struct ieee80211_local *local,
                                 struct ieee80211_chanctx *ctx)
{
        struct ieee80211_chanctx_conf *conf = &ctx->conf;
        struct ieee80211_link_data *link;

        lockdep_assert_wiphy(local->hw.wiphy);

        for_each_sdata_link(local, link) {
                if (rcu_access_pointer(link->conf->chanctx_conf) != conf)
                        continue;
                if (!link->radar_required)
                        continue;
                return true;
        }

        return false;
}

static struct ieee80211_chanctx *
ieee80211_alloc_chanctx(struct ieee80211_local *local,
                        const struct ieee80211_chan_req *chanreq,
                        enum ieee80211_chanctx_mode mode,
                        int radio_idx)
{
        struct ieee80211_chanctx *ctx;

        lockdep_assert_wiphy(local->hw.wiphy);

        ctx = kzalloc(sizeof(*ctx) + local->hw.chanctx_data_size, GFP_KERNEL);
        if (!ctx)
                return NULL;

        INIT_LIST_HEAD(&ctx->assigned_links);
        INIT_LIST_HEAD(&ctx->reserved_links);
        ctx->conf.def = chanreq->oper;
        ctx->conf.ap = chanreq->ap;
        ctx->conf.rx_chains_static = 1;
        ctx->conf.rx_chains_dynamic = 1;
        ctx->mode = mode;
        ctx->conf.radar_enabled = false;
        ctx->conf.radio_idx = radio_idx;
        ctx->radar_detected = false;
        _ieee80211_recalc_chanctx_min_def(local, ctx, NULL, false);

        return ctx;
}

static int ieee80211_add_chanctx(struct ieee80211_local *local,
                                 struct ieee80211_chanctx *ctx)
{
        u32 changed;
        int err;

        lockdep_assert_wiphy(local->hw.wiphy);

        ieee80211_add_wbrf(local, &ctx->conf.def);

        /* turn idle off *before* setting channel -- some drivers need that */
        changed = ieee80211_idle_off(local);
        if (changed)
                ieee80211_hw_config(local, -1, changed);

        err = drv_add_chanctx(local, ctx);
        if (err) {
                ieee80211_recalc_idle(local);
                return err;
        }

        return 0;
}

static struct ieee80211_chanctx *
ieee80211_new_chanctx(struct ieee80211_local *local,
                      const struct ieee80211_chan_req *chanreq,
                      enum ieee80211_chanctx_mode mode,
                      bool assign_on_failure,
                      int radio_idx)
{
        struct ieee80211_chanctx *ctx;
        int err;

        lockdep_assert_wiphy(local->hw.wiphy);

        ctx = ieee80211_alloc_chanctx(local, chanreq, mode, radio_idx);
        if (!ctx)
                return ERR_PTR(-ENOMEM);

        err = ieee80211_add_chanctx(local, ctx);
        if (!assign_on_failure && err) {
                kfree(ctx);
                return ERR_PTR(err);
        }
        /* We ignored a driver error, see _ieee80211_set_active_links */
        WARN_ON_ONCE(err && !local->in_reconfig);

        list_add_rcu(&ctx->list, &local->chanctx_list);
        return ctx;
}

static void ieee80211_del_chanctx(struct ieee80211_local *local,
                                  struct ieee80211_chanctx *ctx,
                                  bool skip_idle_recalc)
{
        lockdep_assert_wiphy(local->hw.wiphy);

        drv_remove_chanctx(local, ctx);

        if (!skip_idle_recalc)
                ieee80211_recalc_idle(local);

        ieee80211_remove_wbrf(local, &ctx->conf.def);
}

static void ieee80211_free_chanctx(struct ieee80211_local *local,
                                   struct ieee80211_chanctx *ctx,
                                   bool skip_idle_recalc)
{
        lockdep_assert_wiphy(local->hw.wiphy);

        WARN_ON_ONCE(ieee80211_chanctx_refcount(local, ctx) != 0);

        list_del_rcu(&ctx->list);
        ieee80211_del_chanctx(local, ctx, skip_idle_recalc);
        kfree_rcu(ctx, rcu_head);
}

void ieee80211_recalc_chanctx_chantype(struct ieee80211_local *local,
                                       struct ieee80211_chanctx *ctx)
{
        struct ieee80211_chanctx_conf *conf = &ctx->conf;
        const struct ieee80211_chan_req *compat = NULL;
        struct ieee80211_link_data *link;
        struct ieee80211_chan_req tmp;
        struct sta_info *sta;

        lockdep_assert_wiphy(local->hw.wiphy);

        for_each_sdata_link(local, link) {
                struct ieee80211_bss_conf *link_conf;

                if (link->sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
                        continue;

                link_conf = link->conf;

                if (rcu_access_pointer(link_conf->chanctx_conf) != conf)
                        continue;

                if (!compat)
                        compat = &link_conf->chanreq;

                compat = ieee80211_chanreq_compatible(&link_conf->chanreq,
                                                      compat, &tmp);
                if (WARN_ON_ONCE(!compat))
                        return;
        }

        if (WARN_ON_ONCE(!compat))
                return;

        /* TDLS peers can sometimes affect the chandef width */
        list_for_each_entry(sta, &local->sta_list, list) {
                struct ieee80211_sub_if_data *sdata = sta->sdata;
                struct ieee80211_chan_req tdls_chanreq = {};
                int tdls_link_id;

                if (!sta->uploaded ||
                    !test_sta_flag(sta, WLAN_STA_TDLS_WIDER_BW) ||
                    !test_sta_flag(sta, WLAN_STA_AUTHORIZED) ||
                    !sta->tdls_chandef.chan)
                        continue;

                tdls_link_id = ieee80211_tdls_sta_link_id(sta);
                link = sdata_dereference(sdata->link[tdls_link_id], sdata);
                if (!link)
                        continue;

                if (rcu_access_pointer(link->conf->chanctx_conf) != conf)
                        continue;

                tdls_chanreq.oper = sta->tdls_chandef;

                /* note this always fills and returns &tmp if compat */
                compat = ieee80211_chanreq_compatible(&tdls_chanreq,
                                                      compat, &tmp);
                if (WARN_ON_ONCE(!compat))
                        return;
        }

        ieee80211_change_chanctx(local, ctx, ctx, compat);
}

static void ieee80211_recalc_radar_chanctx(struct ieee80211_local *local,
                                           struct ieee80211_chanctx *chanctx)
{
        bool radar_enabled;

        lockdep_assert_wiphy(local->hw.wiphy);

        radar_enabled = ieee80211_chanctx_radar_required(local, chanctx);

        if (radar_enabled == chanctx->conf.radar_enabled)
                return;

        chanctx->conf.radar_enabled = radar_enabled;

        drv_change_chanctx(local, chanctx, IEEE80211_CHANCTX_CHANGE_RADAR);
}

static int ieee80211_assign_link_chanctx(struct ieee80211_link_data *link,
                                         struct ieee80211_chanctx *new_ctx,
                                         bool assign_on_failure)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *conf;
        struct ieee80211_chanctx *curr_ctx = NULL;
        bool new_idle;
        int ret;

        if (WARN_ON(sdata->vif.type == NL80211_IFTYPE_NAN))
                return -EOPNOTSUPP;

        conf = rcu_dereference_protected(link->conf->chanctx_conf,
                                         lockdep_is_held(&local->hw.wiphy->mtx));

        if (conf && !local->in_reconfig) {
                curr_ctx = container_of(conf, struct ieee80211_chanctx, conf);

                drv_unassign_vif_chanctx(local, sdata, link->conf, curr_ctx);
                conf = NULL;
                list_del(&link->assigned_chanctx_list);
        }

        if (new_ctx) {
                /* recalc considering the link we'll use it for now */
                ieee80211_recalc_chanctx_min_def(local, new_ctx, link, false);

                ret = drv_assign_vif_chanctx(local, sdata, link->conf, new_ctx);
                if (assign_on_failure || !ret) {
                        /* Need to continue, see _ieee80211_set_active_links */
                        WARN_ON_ONCE(ret && !local->in_reconfig);
                        ret = 0;

                        /* succeeded, so commit it to the data structures */
                        conf = &new_ctx->conf;
                        if (!local->in_reconfig)
                                list_add(&link->assigned_chanctx_list,
                                         &new_ctx->assigned_links);
                }
        } else {
                ret = 0;
        }

        rcu_assign_pointer(link->conf->chanctx_conf, conf);

        if (curr_ctx && ieee80211_chanctx_num_assigned(local, curr_ctx) > 0) {
                ieee80211_recalc_chanctx_chantype(local, curr_ctx);
                ieee80211_recalc_smps_chanctx(local, curr_ctx);
                ieee80211_recalc_radar_chanctx(local, curr_ctx);
                ieee80211_recalc_chanctx_min_def(local, curr_ctx, NULL, false);
        }

        if (new_ctx && ieee80211_chanctx_num_assigned(local, new_ctx) > 0) {
                ieee80211_recalc_txpower(link, false);
                ieee80211_recalc_chanctx_min_def(local, new_ctx, NULL, false);
        }

        if (conf) {
                new_idle = false;
        } else {
                struct ieee80211_link_data *tmp;

                new_idle = true;
                for_each_sdata_link(local, tmp) {
                        if (rcu_access_pointer(tmp->conf->chanctx_conf)) {
                                new_idle = false;
                                break;
                        }
                }
        }

        if (new_idle != sdata->vif.cfg.idle) {
                sdata->vif.cfg.idle = new_idle;

                if (sdata->vif.type != NL80211_IFTYPE_P2P_DEVICE &&
                    sdata->vif.type != NL80211_IFTYPE_MONITOR)
                        ieee80211_vif_cfg_change_notify(sdata, BSS_CHANGED_IDLE);
        }

        ieee80211_check_fast_xmit_iface(sdata);

        return ret;
}

void ieee80211_recalc_smps_chanctx(struct ieee80211_local *local,
                                   struct ieee80211_chanctx *chanctx)
{
        struct ieee80211_sub_if_data *sdata;
        u8 rx_chains_static, rx_chains_dynamic;
        struct ieee80211_link_data *link;

        lockdep_assert_wiphy(local->hw.wiphy);

        rx_chains_static = 1;
        rx_chains_dynamic = 1;

        for_each_sdata_link(local, link) {
                u8 needed_static, needed_dynamic;

                switch (link->sdata->vif.type) {
                case NL80211_IFTYPE_STATION:
                        if (!link->sdata->u.mgd.associated)
                                continue;
                        break;
                case NL80211_IFTYPE_MONITOR:
                        if (!ieee80211_hw_check(&local->hw, NO_VIRTUAL_MONITOR))
                                continue;
                        break;
                case NL80211_IFTYPE_AP:
                case NL80211_IFTYPE_ADHOC:
                case NL80211_IFTYPE_MESH_POINT:
                case NL80211_IFTYPE_OCB:
                        break;
                default:
                        continue;
                }

                if (rcu_access_pointer(link->conf->chanctx_conf) != &chanctx->conf)
                        continue;

                if (link->sdata->vif.type == NL80211_IFTYPE_MONITOR) {
                        rx_chains_dynamic = rx_chains_static = local->rx_chains;
                        break;
                }

                switch (link->smps_mode) {
                default:
                        WARN_ONCE(1, "Invalid SMPS mode %d\n",
                                  link->smps_mode);
                        fallthrough;
                case IEEE80211_SMPS_OFF:
                        needed_static = link->needed_rx_chains;
                        needed_dynamic = link->needed_rx_chains;
                        break;
                case IEEE80211_SMPS_DYNAMIC:
                        needed_static = 1;
                        needed_dynamic = link->needed_rx_chains;
                        break;
                case IEEE80211_SMPS_STATIC:
                        needed_static = 1;
                        needed_dynamic = 1;
                        break;
                }

                rx_chains_static = max(rx_chains_static, needed_static);
                rx_chains_dynamic = max(rx_chains_dynamic, needed_dynamic);
        }

        /* Disable SMPS for the monitor interface */
        sdata = wiphy_dereference(local->hw.wiphy, local->monitor_sdata);
        if (sdata &&
            rcu_access_pointer(sdata->vif.bss_conf.chanctx_conf) == &chanctx->conf)
                rx_chains_dynamic = rx_chains_static = local->rx_chains;

        if (rx_chains_static == chanctx->conf.rx_chains_static &&
            rx_chains_dynamic == chanctx->conf.rx_chains_dynamic)
                return;

        chanctx->conf.rx_chains_static = rx_chains_static;
        chanctx->conf.rx_chains_dynamic = rx_chains_dynamic;
        drv_change_chanctx(local, chanctx, IEEE80211_CHANCTX_CHANGE_RX_CHAINS);
}

static void
__ieee80211_link_copy_chanctx_to_vlans(struct ieee80211_link_data *link,
                                       bool clear)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        unsigned int link_id = link->link_id;
        struct ieee80211_bss_conf *link_conf = link->conf;
        struct ieee80211_local *local __maybe_unused = sdata->local;
        struct ieee80211_sub_if_data *vlan;
        struct ieee80211_chanctx_conf *conf;

        if (WARN_ON(sdata->vif.type != NL80211_IFTYPE_AP))
                return;

        lockdep_assert_wiphy(local->hw.wiphy);

        /* Check that conf exists, even when clearing this function
         * must be called with the AP's channel context still there
         * as it would otherwise cause VLANs to have an invalid
         * channel context pointer for a while, possibly pointing
         * to a channel context that has already been freed.
         */
        conf = rcu_dereference_protected(link_conf->chanctx_conf,
                                         lockdep_is_held(&local->hw.wiphy->mtx));
        WARN_ON(!conf);

        if (clear)
                conf = NULL;

        list_for_each_entry(vlan, &sdata->u.ap.vlans, u.vlan.list) {
                struct ieee80211_bss_conf *vlan_conf;

                vlan_conf = wiphy_dereference(local->hw.wiphy,
                                              vlan->vif.link_conf[link_id]);
                if (WARN_ON(!vlan_conf))
                        continue;

                rcu_assign_pointer(vlan_conf->chanctx_conf, conf);
        }
}

void ieee80211_link_copy_chanctx_to_vlans(struct ieee80211_link_data *link,
                                          bool clear)
{
        struct ieee80211_local *local = link->sdata->local;

        lockdep_assert_wiphy(local->hw.wiphy);

        __ieee80211_link_copy_chanctx_to_vlans(link, clear);
}

void ieee80211_link_unreserve_chanctx(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_chanctx *ctx = link->reserved_chanctx;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        if (WARN_ON(!ctx))
                return;

        list_del(&link->reserved_chanctx_list);
        link->reserved_chanctx = NULL;

        if (ieee80211_chanctx_refcount(sdata->local, ctx) == 0) {
                if (ctx->replace_state == IEEE80211_CHANCTX_REPLACES_OTHER) {
                        if (WARN_ON(!ctx->replace_ctx))
                                return;

                        WARN_ON(ctx->replace_ctx->replace_state !=
                                IEEE80211_CHANCTX_WILL_BE_REPLACED);
                        WARN_ON(ctx->replace_ctx->replace_ctx != ctx);

                        ctx->replace_ctx->replace_ctx = NULL;
                        ctx->replace_ctx->replace_state =
                                        IEEE80211_CHANCTX_REPLACE_NONE;

                        list_del_rcu(&ctx->list);
                        kfree_rcu(ctx, rcu_head);
                } else {
                        ieee80211_free_chanctx(sdata->local, ctx, false);
                }
        }
}

static struct ieee80211_chanctx *
ieee80211_replace_chanctx(struct ieee80211_local *local,
                          const struct ieee80211_chan_req *chanreq,
                          enum ieee80211_chanctx_mode mode,
                          struct ieee80211_chanctx *curr_ctx)
{
        struct ieee80211_chanctx *new_ctx, *ctx;
        struct wiphy *wiphy = local->hw.wiphy;
        const struct wiphy_radio *radio;

        if (!curr_ctx || (curr_ctx->replace_state ==
                          IEEE80211_CHANCTX_WILL_BE_REPLACED) ||
            !list_empty(&curr_ctx->reserved_links)) {
                /*
                 * Another link already requested this context for a
                 * reservation. Find another one hoping all links assigned
                 * to it will also switch soon enough.
                 *
                 * TODO: This needs a little more work as some cases
                 * (more than 2 chanctx capable devices) may fail which could
                 * otherwise succeed provided some channel context juggling was
                 * performed.
                 *
                 * Consider ctx1..3, link1..6, each ctx has 2 links. link1 and
                 * link2 from ctx1 request new different chandefs starting 2
                 * in-place reservations with ctx4 and ctx5 replacing ctx1 and
                 * ctx2 respectively. Next link5 and link6 from ctx3 reserve
                 * ctx4. If link3 and link4 remain on ctx2 as they are then this
                 * fails unless `replace_ctx` from ctx5 is replaced with ctx3.
                 */
                list_for_each_entry(ctx, &local->chanctx_list, list) {
                        if (ctx->replace_state !=
                            IEEE80211_CHANCTX_REPLACE_NONE)
                                continue;

                        if (!list_empty(&ctx->reserved_links))
                                continue;

                        if (ctx->conf.radio_idx >= 0) {
                                radio = &wiphy->radio[ctx->conf.radio_idx];
                                if (!cfg80211_radio_chandef_valid(radio, &chanreq->oper))
                                        continue;
                        }

                        curr_ctx = ctx;
                        break;
                }
        }

        /*
         * If that's true then all available contexts already have reservations
         * and cannot be used.
         */
        if (!curr_ctx || (curr_ctx->replace_state ==
                          IEEE80211_CHANCTX_WILL_BE_REPLACED) ||
            !list_empty(&curr_ctx->reserved_links))
                return ERR_PTR(-EBUSY);

        new_ctx = ieee80211_alloc_chanctx(local, chanreq, mode, -1);
        if (!new_ctx)
                return ERR_PTR(-ENOMEM);

        new_ctx->replace_ctx = curr_ctx;
        new_ctx->replace_state = IEEE80211_CHANCTX_REPLACES_OTHER;

        curr_ctx->replace_ctx = new_ctx;
        curr_ctx->replace_state = IEEE80211_CHANCTX_WILL_BE_REPLACED;

        list_add_rcu(&new_ctx->list, &local->chanctx_list);

        return new_ctx;
}

static bool
ieee80211_find_available_radio(struct ieee80211_local *local,
                               const struct ieee80211_chan_req *chanreq,
                               u32 radio_mask, int *radio_idx)
{
        struct wiphy *wiphy = local->hw.wiphy;
        const struct wiphy_radio *radio;
        int i;

        *radio_idx = -1;
        if (!wiphy->n_radio)
                return true;

        for (i = 0; i < wiphy->n_radio; i++) {
                if (!(radio_mask & BIT(i)))
                        continue;

                radio = &wiphy->radio[i];
                if (!cfg80211_radio_chandef_valid(radio, &chanreq->oper))
                        continue;

                if (!ieee80211_can_create_new_chanctx(local, i))
                        continue;

                *radio_idx = i;
                return true;
        }

        return false;
}

int ieee80211_link_reserve_chanctx(struct ieee80211_link_data *link,
                                   const struct ieee80211_chan_req *chanreq,
                                   enum ieee80211_chanctx_mode mode,
                                   bool radar_required)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx *new_ctx, *curr_ctx;
        int radio_idx;

        lockdep_assert_wiphy(local->hw.wiphy);

        curr_ctx = ieee80211_link_get_chanctx(link);
        if (curr_ctx && !local->ops->switch_vif_chanctx)
                return -EOPNOTSUPP;

        new_ctx = ieee80211_find_reservation_chanctx(local, chanreq, mode);
        if (!new_ctx) {
                if (ieee80211_can_create_new_chanctx(local, -1) &&
                    ieee80211_find_available_radio(local, chanreq,
                                                   sdata->wdev.radio_mask,
                                                   &radio_idx))
                        new_ctx = ieee80211_new_chanctx(local, chanreq, mode,
                                                        false, radio_idx);
                else
                        new_ctx = ieee80211_replace_chanctx(local, chanreq,
                                                            mode, curr_ctx);
                if (IS_ERR(new_ctx))
                        return PTR_ERR(new_ctx);
        }

        list_add(&link->reserved_chanctx_list, &new_ctx->reserved_links);
        link->reserved_chanctx = new_ctx;
        link->reserved = *chanreq;
        link->reserved_radar_required = radar_required;
        link->reserved_ready = false;

        return 0;
}

static void
ieee80211_link_chanctx_reservation_complete(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;

        switch (sdata->vif.type) {
        case NL80211_IFTYPE_ADHOC:
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_MESH_POINT:
        case NL80211_IFTYPE_OCB:
                wiphy_work_queue(sdata->local->hw.wiphy,
                                 &link->csa.finalize_work);
                break;
        case NL80211_IFTYPE_STATION:
                wiphy_delayed_work_queue(sdata->local->hw.wiphy,
                                         &link->u.mgd.csa.switch_work, 0);
                break;
        case NL80211_IFTYPE_UNSPECIFIED:
        case NL80211_IFTYPE_AP_VLAN:
        case NL80211_IFTYPE_WDS:
        case NL80211_IFTYPE_MONITOR:
        case NL80211_IFTYPE_P2P_CLIENT:
        case NL80211_IFTYPE_P2P_GO:
        case NL80211_IFTYPE_P2P_DEVICE:
        case NL80211_IFTYPE_NAN:
        case NUM_NL80211_IFTYPES:
                WARN_ON(1);
                break;
        }
}

static void
ieee80211_link_update_chanreq(struct ieee80211_link_data *link,
                              const struct ieee80211_chan_req *chanreq)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        unsigned int link_id = link->link_id;
        struct ieee80211_sub_if_data *vlan;

        link->conf->chanreq = *chanreq;

        if (sdata->vif.type != NL80211_IFTYPE_AP)
                return;

        list_for_each_entry(vlan, &sdata->u.ap.vlans, u.vlan.list) {
                struct ieee80211_bss_conf *vlan_conf;

                vlan_conf = wiphy_dereference(sdata->local->hw.wiphy,
                                              vlan->vif.link_conf[link_id]);
                if (WARN_ON(!vlan_conf))
                        continue;

                vlan_conf->chanreq = *chanreq;
        }
}

static int
ieee80211_link_use_reserved_reassign(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_bss_conf *link_conf = link->conf;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_vif_chanctx_switch vif_chsw[1] = {};
        struct ieee80211_chanctx *old_ctx, *new_ctx;
        const struct ieee80211_chan_req *chanreq;
        struct ieee80211_chan_req tmp;
        u64 changed = 0;
        int err;

        lockdep_assert_wiphy(local->hw.wiphy);

        new_ctx = link->reserved_chanctx;
        old_ctx = ieee80211_link_get_chanctx(link);

        if (WARN_ON(!link->reserved_ready))
                return -EBUSY;

        if (WARN_ON(!new_ctx))
                return -EINVAL;

        if (WARN_ON(!old_ctx))
                return -EINVAL;

        if (WARN_ON(new_ctx->replace_state ==
                    IEEE80211_CHANCTX_REPLACES_OTHER))
                return -EINVAL;

        chanreq = ieee80211_chanctx_non_reserved_chandef(local, new_ctx,
                                                         &link->reserved,
                                                         &tmp);
        if (WARN_ON(!chanreq))
                return -EINVAL;

        if (link_conf->chanreq.oper.width != link->reserved.oper.width)
                changed = BSS_CHANGED_BANDWIDTH;

        ieee80211_link_update_chanreq(link, &link->reserved);

        _ieee80211_change_chanctx(local, new_ctx, old_ctx, chanreq, link);

        vif_chsw[0].vif = &sdata->vif;
        vif_chsw[0].old_ctx = &old_ctx->conf;
        vif_chsw[0].new_ctx = &new_ctx->conf;
        vif_chsw[0].link_conf = link->conf;

        list_del(&link->reserved_chanctx_list);
        link->reserved_chanctx = NULL;

        err = drv_switch_vif_chanctx(local, vif_chsw, 1,
                                     CHANCTX_SWMODE_REASSIGN_VIF);
        if (err) {
                if (ieee80211_chanctx_refcount(local, new_ctx) == 0)
                        ieee80211_free_chanctx(local, new_ctx, false);

                goto out;
        }

        link->radar_required = link->reserved_radar_required;
        list_move(&link->assigned_chanctx_list, &new_ctx->assigned_links);
        rcu_assign_pointer(link_conf->chanctx_conf, &new_ctx->conf);

        if (sdata->vif.type == NL80211_IFTYPE_AP)
                __ieee80211_link_copy_chanctx_to_vlans(link, false);

        ieee80211_check_fast_xmit_iface(sdata);

        if (ieee80211_chanctx_refcount(local, old_ctx) == 0)
                ieee80211_free_chanctx(local, old_ctx, false);

        ieee80211_recalc_chanctx_min_def(local, new_ctx, NULL, false);
        ieee80211_recalc_smps_chanctx(local, new_ctx);
        ieee80211_recalc_radar_chanctx(local, new_ctx);

        if (changed)
                ieee80211_link_info_change_notify(sdata, link, changed);

out:
        ieee80211_link_chanctx_reservation_complete(link);
        return err;
}

static int
ieee80211_link_use_reserved_assign(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx *old_ctx, *new_ctx;
        const struct ieee80211_chan_req *chanreq;
        struct ieee80211_chan_req tmp;
        int err;

        old_ctx = ieee80211_link_get_chanctx(link);
        new_ctx = link->reserved_chanctx;

        if (WARN_ON(!link->reserved_ready))
                return -EINVAL;

        if (WARN_ON(old_ctx))
                return -EINVAL;

        if (WARN_ON(!new_ctx))
                return -EINVAL;

        if (WARN_ON(new_ctx->replace_state ==
                    IEEE80211_CHANCTX_REPLACES_OTHER))
                return -EINVAL;

        chanreq = ieee80211_chanctx_non_reserved_chandef(local, new_ctx,
                                                         &link->reserved,
                                                         &tmp);
        if (WARN_ON(!chanreq))
                return -EINVAL;

        ieee80211_change_chanctx(local, new_ctx, new_ctx, chanreq);

        list_del(&link->reserved_chanctx_list);
        link->reserved_chanctx = NULL;

        err = ieee80211_assign_link_chanctx(link, new_ctx, false);
        if (err) {
                if (ieee80211_chanctx_refcount(local, new_ctx) == 0)
                        ieee80211_free_chanctx(local, new_ctx, false);

                goto out;
        }

out:
        ieee80211_link_chanctx_reservation_complete(link);
        return err;
}

static bool
ieee80211_link_has_in_place_reservation(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_chanctx *old_ctx, *new_ctx;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        new_ctx = link->reserved_chanctx;
        old_ctx = ieee80211_link_get_chanctx(link);

        if (!old_ctx)
                return false;

        if (WARN_ON(!new_ctx))
                return false;

        if (old_ctx->replace_state != IEEE80211_CHANCTX_WILL_BE_REPLACED)
                return false;

        if (new_ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
                return false;

        return true;
}

static int ieee80211_chsw_switch_vifs(struct ieee80211_local *local,
                                      int n_vifs)
{
        struct ieee80211_vif_chanctx_switch *vif_chsw;
        struct ieee80211_link_data *link;
        struct ieee80211_chanctx *ctx, *old_ctx;
        int i, err;

        lockdep_assert_wiphy(local->hw.wiphy);

        vif_chsw = kcalloc(n_vifs, sizeof(vif_chsw[0]), GFP_KERNEL);
        if (!vif_chsw)
                return -ENOMEM;

        i = 0;
        list_for_each_entry(ctx, &local->chanctx_list, list) {
                if (ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
                        continue;

                if (WARN_ON(!ctx->replace_ctx)) {
                        err = -EINVAL;
                        goto out;
                }

                list_for_each_entry(link, &ctx->reserved_links,
                                    reserved_chanctx_list) {
                        if (!ieee80211_link_has_in_place_reservation(link))
                                continue;

                        old_ctx = ieee80211_link_get_chanctx(link);
                        vif_chsw[i].vif = &link->sdata->vif;
                        vif_chsw[i].old_ctx = &old_ctx->conf;
                        vif_chsw[i].new_ctx = &ctx->conf;
                        vif_chsw[i].link_conf = link->conf;

                        i++;
                }
        }

        err = drv_switch_vif_chanctx(local, vif_chsw, n_vifs,
                                     CHANCTX_SWMODE_SWAP_CONTEXTS);

out:
        kfree(vif_chsw);
        return err;
}

static int ieee80211_chsw_switch_ctxs(struct ieee80211_local *local)
{
        struct ieee80211_chanctx *ctx;
        int err;

        lockdep_assert_wiphy(local->hw.wiphy);

        list_for_each_entry(ctx, &local->chanctx_list, list) {
                if (ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
                        continue;

                if (!list_empty(&ctx->replace_ctx->assigned_links))
                        continue;

                ieee80211_del_chanctx(local, ctx->replace_ctx, false);
                err = ieee80211_add_chanctx(local, ctx);
                if (err)
                        goto err;
        }

        return 0;

err:
        WARN_ON(ieee80211_add_chanctx(local, ctx));
        list_for_each_entry_continue_reverse(ctx, &local->chanctx_list, list) {
                if (ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
                        continue;

                if (!list_empty(&ctx->replace_ctx->assigned_links))
                        continue;

                ieee80211_del_chanctx(local, ctx, false);
                WARN_ON(ieee80211_add_chanctx(local, ctx->replace_ctx));
        }

        return err;
}

static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local)
{
        struct ieee80211_chanctx *ctx, *ctx_tmp, *old_ctx;
        int err, n_assigned, n_reserved, n_ready;
        int n_ctx = 0, n_vifs_switch = 0, n_vifs_assign = 0, n_vifs_ctxless = 0;

        lockdep_assert_wiphy(local->hw.wiphy);

        /*
         * If there are 2 independent pairs of channel contexts performing
         * cross-switch of their vifs this code will still wait until both are
         * ready even though it could be possible to switch one before the
         * other is ready.
         *
         * For practical reasons and code simplicity just do a single huge
         * switch.
         */

        /*
         * Verify if the reservation is still feasible.
         *  - if it's not then disconnect
         *  - if it is but not all vifs necessary are ready then defer
         */

        list_for_each_entry(ctx, &local->chanctx_list, list) {
                struct ieee80211_link_data *link;

                if (ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
                        continue;

                if (WARN_ON(!ctx->replace_ctx)) {
                        err = -EINVAL;
                        goto err;
                }

                n_ctx++;

                n_assigned = 0;
                n_reserved = 0;
                n_ready = 0;

                list_for_each_entry(link, &ctx->replace_ctx->assigned_links,
                                    assigned_chanctx_list) {
                        n_assigned++;
                        if (link->reserved_chanctx) {
                                n_reserved++;
                                if (link->reserved_ready)
                                        n_ready++;
                        }
                }

                if (n_assigned != n_reserved) {
                        if (n_ready == n_reserved) {
                                wiphy_info(local->hw.wiphy,
                                           "channel context reservation cannot be finalized because some interfaces aren't switching\n");
                                err = -EBUSY;
                                goto err;
                        }

                        return -EAGAIN;
                }

                ctx->conf.radar_enabled = false;
                list_for_each_entry(link, &ctx->reserved_links,
                                    reserved_chanctx_list) {
                        if (ieee80211_link_has_in_place_reservation(link) &&
                            !link->reserved_ready)
                                return -EAGAIN;

                        old_ctx = ieee80211_link_get_chanctx(link);
                        if (old_ctx) {
                                if (old_ctx->replace_state ==
                                    IEEE80211_CHANCTX_WILL_BE_REPLACED)
                                        n_vifs_switch++;
                                else
                                        n_vifs_assign++;
                        } else {
                                n_vifs_ctxless++;
                        }

                        if (link->reserved_radar_required)
                                ctx->conf.radar_enabled = true;
                }
        }

        if (WARN_ON(n_ctx == 0) ||
            WARN_ON(n_vifs_switch == 0 &&
                    n_vifs_assign == 0 &&
                    n_vifs_ctxless == 0)) {
                err = -EINVAL;
                goto err;
        }

        /* update station rate control and min width before switch */
        list_for_each_entry(ctx, &local->chanctx_list, list) {
                struct ieee80211_link_data *link;

                if (ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
                        continue;

                if (WARN_ON(!ctx->replace_ctx)) {
                        err = -EINVAL;
                        goto err;
                }

                list_for_each_entry(link, &ctx->reserved_links,
                                    reserved_chanctx_list) {
                        if (!ieee80211_link_has_in_place_reservation(link))
                                continue;

                        ieee80211_chan_bw_change(local,
                                                 ieee80211_link_get_chanctx(link),
                                                 true, true);
                }

                ieee80211_recalc_chanctx_min_def(local, ctx, NULL, true);
        }

        /*
         * All necessary vifs are ready. Perform the switch now depending on
         * reservations and driver capabilities.
         */

        if (n_vifs_switch > 0) {
                err = ieee80211_chsw_switch_vifs(local, n_vifs_switch);
                if (err)
                        goto err;
        }

        if (n_vifs_assign > 0 || n_vifs_ctxless > 0) {
                err = ieee80211_chsw_switch_ctxs(local);
                if (err)
                        goto err;
        }

        /*
         * Update all structures, values and pointers to point to new channel
         * context(s).
         */
        list_for_each_entry(ctx, &local->chanctx_list, list) {
                struct ieee80211_link_data *link, *link_tmp;

                if (ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
                        continue;

                if (WARN_ON(!ctx->replace_ctx)) {
                        err = -EINVAL;
                        goto err;
                }

                list_for_each_entry(link, &ctx->reserved_links,
                                    reserved_chanctx_list) {
                        struct ieee80211_sub_if_data *sdata = link->sdata;
                        struct ieee80211_bss_conf *link_conf = link->conf;
                        u64 changed = 0;

                        if (!ieee80211_link_has_in_place_reservation(link))
                                continue;

                        rcu_assign_pointer(link_conf->chanctx_conf,
                                           &ctx->conf);

                        if (sdata->vif.type == NL80211_IFTYPE_AP)
                                __ieee80211_link_copy_chanctx_to_vlans(link,
                                                                       false);

                        ieee80211_check_fast_xmit_iface(sdata);

                        link->radar_required = link->reserved_radar_required;

                        if (link_conf->chanreq.oper.width != link->reserved.oper.width)
                                changed = BSS_CHANGED_BANDWIDTH;

                        ieee80211_link_update_chanreq(link, &link->reserved);
                        if (changed)
                                ieee80211_link_info_change_notify(sdata,
                                                                  link,
                                                                  changed);

                        ieee80211_recalc_txpower(link, false);
                }

                ieee80211_recalc_chanctx_chantype(local, ctx);
                ieee80211_recalc_smps_chanctx(local, ctx);
                ieee80211_recalc_radar_chanctx(local, ctx);
                ieee80211_recalc_chanctx_min_def(local, ctx, NULL, false);

                list_for_each_entry_safe(link, link_tmp, &ctx->reserved_links,
                                         reserved_chanctx_list) {
                        if (ieee80211_link_get_chanctx(link) != ctx)
                                continue;

                        list_del(&link->reserved_chanctx_list);
                        list_move(&link->assigned_chanctx_list,
                                  &ctx->assigned_links);
                        link->reserved_chanctx = NULL;

                        ieee80211_link_chanctx_reservation_complete(link);
                        ieee80211_chan_bw_change(local, ctx, false, false);
                }

                /*
                 * This context might have been a dependency for an already
                 * ready re-assign reservation interface that was deferred. Do
                 * not propagate error to the caller though. The in-place
                 * reservation for originally requested interface has already
                 * succeeded at this point.
                 */
                list_for_each_entry_safe(link, link_tmp, &ctx->reserved_links,
                                         reserved_chanctx_list) {
                        if (WARN_ON(ieee80211_link_has_in_place_reservation(link)))
                                continue;

                        if (WARN_ON(link->reserved_chanctx != ctx))
                                continue;

                        if (!link->reserved_ready)
                                continue;

                        if (ieee80211_link_get_chanctx(link))
                                err = ieee80211_link_use_reserved_reassign(link);
                        else
                                err = ieee80211_link_use_reserved_assign(link);

                        if (err) {
                                link_info(link,
                                          "failed to finalize (re-)assign reservation (err=%d)\n",
                                          err);
                                ieee80211_link_unreserve_chanctx(link);
                                cfg80211_stop_iface(local->hw.wiphy,
                                                    &link->sdata->wdev,
                                                    GFP_KERNEL);
                        }
                }
        }

        /*
         * Finally free old contexts
         */

        list_for_each_entry_safe(ctx, ctx_tmp, &local->chanctx_list, list) {
                if (ctx->replace_state != IEEE80211_CHANCTX_WILL_BE_REPLACED)
                        continue;

                ctx->replace_ctx->replace_ctx = NULL;
                ctx->replace_ctx->replace_state =
                                IEEE80211_CHANCTX_REPLACE_NONE;

                list_del_rcu(&ctx->list);
                kfree_rcu(ctx, rcu_head);
        }

        return 0;

err:
        list_for_each_entry(ctx, &local->chanctx_list, list) {
                struct ieee80211_link_data *link, *link_tmp;

                if (ctx->replace_state != IEEE80211_CHANCTX_REPLACES_OTHER)
                        continue;

                list_for_each_entry_safe(link, link_tmp, &ctx->reserved_links,
                                         reserved_chanctx_list) {
                        ieee80211_link_unreserve_chanctx(link);
                        ieee80211_link_chanctx_reservation_complete(link);
                }
        }

        return err;
}

void __ieee80211_link_release_channel(struct ieee80211_link_data *link,
                                      bool skip_idle_recalc)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_bss_conf *link_conf = link->conf;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *conf;
        struct ieee80211_chanctx *ctx;
        bool use_reserved_switch = false;

        lockdep_assert_wiphy(local->hw.wiphy);

        conf = rcu_dereference_protected(link_conf->chanctx_conf,
                                         lockdep_is_held(&local->hw.wiphy->mtx));
        if (!conf)
                return;

        ctx = container_of(conf, struct ieee80211_chanctx, conf);

        if (link->reserved_chanctx) {
                if (link->reserved_chanctx->replace_state == IEEE80211_CHANCTX_REPLACES_OTHER &&
                    ieee80211_chanctx_num_reserved(local, link->reserved_chanctx) > 1)
                        use_reserved_switch = true;

                ieee80211_link_unreserve_chanctx(link);
        }

        ieee80211_assign_link_chanctx(link, NULL, false);
        if (ieee80211_chanctx_refcount(local, ctx) == 0)
                ieee80211_free_chanctx(local, ctx, skip_idle_recalc);

        link->radar_required = false;

        /* Unreserving may ready an in-place reservation. */
        if (use_reserved_switch)
                ieee80211_vif_use_reserved_switch(local);
}

int _ieee80211_link_use_channel(struct ieee80211_link_data *link,
                                const struct ieee80211_chan_req *chanreq,
                                enum ieee80211_chanctx_mode mode,
                                bool assign_on_failure)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx *ctx;
        u8 radar_detect_width = 0;
        bool reserved = false;
        int radio_idx;
        int ret;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!ieee80211_vif_link_active(&sdata->vif, link->link_id)) {
                ieee80211_link_update_chanreq(link, chanreq);
                return 0;
        }

        ret = cfg80211_chandef_dfs_required(local->hw.wiphy,
                                            &chanreq->oper,
                                            sdata->wdev.iftype);
        if (ret < 0)
                goto out;
        if (ret > 0)
                radar_detect_width = BIT(chanreq->oper.width);

        link->radar_required = ret;

        ret = ieee80211_check_combinations(sdata, &chanreq->oper, mode,
                                           radar_detect_width, -1);
        if (ret < 0)
                goto out;

        if (!local->in_reconfig)
                __ieee80211_link_release_channel(link, false);

        ctx = ieee80211_find_chanctx(local, link, chanreq, mode);
        /* Note: context is now reserved */
        if (ctx)
                reserved = true;
        else if (!ieee80211_find_available_radio(local, chanreq,
                                                 sdata->wdev.radio_mask,
                                                 &radio_idx))
                ctx = ERR_PTR(-EBUSY);
        else
                ctx = ieee80211_new_chanctx(local, chanreq, mode,
                                            assign_on_failure, radio_idx);
        if (IS_ERR(ctx)) {
                ret = PTR_ERR(ctx);
                goto out;
        }

        ieee80211_link_update_chanreq(link, chanreq);

        ret = ieee80211_assign_link_chanctx(link, ctx, assign_on_failure);

        if (reserved) {
                /* remove reservation */
                WARN_ON(link->reserved_chanctx != ctx);
                link->reserved_chanctx = NULL;
                list_del(&link->reserved_chanctx_list);
        }

        if (ret) {
                /* if assign fails refcount stays the same */
                if (ieee80211_chanctx_refcount(local, ctx) == 0)
                        ieee80211_free_chanctx(local, ctx, false);
                goto out;
        }

        ieee80211_recalc_smps_chanctx(local, ctx);
        ieee80211_recalc_radar_chanctx(local, ctx);
 out:
        if (ret)
                link->radar_required = false;

        return ret;
}

int ieee80211_link_use_reserved_context(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx *new_ctx;
        struct ieee80211_chanctx *old_ctx;
        int err;

        lockdep_assert_wiphy(local->hw.wiphy);

        new_ctx = link->reserved_chanctx;
        old_ctx = ieee80211_link_get_chanctx(link);

        if (WARN_ON(!new_ctx))
                return -EINVAL;

        if (WARN_ON(new_ctx->replace_state ==
                    IEEE80211_CHANCTX_WILL_BE_REPLACED))
                return -EINVAL;

        if (WARN_ON(link->reserved_ready))
                return -EINVAL;

        link->reserved_ready = true;

        if (new_ctx->replace_state == IEEE80211_CHANCTX_REPLACE_NONE) {
                if (old_ctx)
                        return ieee80211_link_use_reserved_reassign(link);

                return ieee80211_link_use_reserved_assign(link);
        }

        /*
         * In-place reservation may need to be finalized now either if:
         *  a) sdata is taking part in the swapping itself and is the last one
         *  b) sdata has switched with a re-assign reservation to an existing
         *     context readying in-place switching of old_ctx
         *
         * In case of (b) do not propagate the error up because the requested
         * sdata already switched successfully. Just spill an extra warning.
         * The ieee80211_vif_use_reserved_switch() already stops all necessary
         * interfaces upon failure.
         */
        if ((old_ctx &&
             old_ctx->replace_state == IEEE80211_CHANCTX_WILL_BE_REPLACED) ||
            new_ctx->replace_state == IEEE80211_CHANCTX_REPLACES_OTHER) {
                err = ieee80211_vif_use_reserved_switch(local);
                if (err && err != -EAGAIN) {
                        if (new_ctx->replace_state ==
                            IEEE80211_CHANCTX_REPLACES_OTHER)
                                return err;

                        wiphy_info(local->hw.wiphy,
                                   "depending in-place reservation failed (err=%d)\n",
                                   err);
                }
        }

        return 0;
}

/*
 * This is similar to ieee80211_chanctx_compatible(), but rechecks
 * against all the links actually using it (except the one that's
 * passed, since that one is changing).
 * This is done in order to allow changes to the AP's bandwidth for
 * wider bandwidth OFDMA purposes, which wouldn't be treated as
 * compatible by ieee80211_chanctx_recheck() but is OK if the link
 * requesting the update is the only one using it.
 */
static const struct ieee80211_chan_req *
ieee80211_chanctx_recheck(struct ieee80211_local *local,
                          struct ieee80211_link_data *skip_link,
                          struct ieee80211_chanctx *ctx,
                          const struct ieee80211_chan_req *req,
                          struct ieee80211_chan_req *tmp)
{
        const struct ieee80211_chan_req *ret = req;
        struct ieee80211_link_data *link;

        lockdep_assert_wiphy(local->hw.wiphy);

        for_each_sdata_link(local, link) {
                if (link == skip_link)
                        continue;

                if (rcu_access_pointer(link->conf->chanctx_conf) == &ctx->conf) {
                        ret = ieee80211_chanreq_compatible(ret,
                                                           &link->conf->chanreq,
                                                           tmp);
                        if (!ret)
                                return NULL;
                }

                if (link->reserved_chanctx == ctx) {
                        ret = ieee80211_chanreq_compatible(ret,
                                                           &link->reserved,
                                                           tmp);
                        if (!ret)
                                return NULL;
                }
        }

        *tmp = *ret;
        return tmp;
}

int ieee80211_link_change_chanreq(struct ieee80211_link_data *link,
                                  const struct ieee80211_chan_req *chanreq,
                                  u64 *changed)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_bss_conf *link_conf = link->conf;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *conf;
        struct ieee80211_chanctx *ctx;
        const struct ieee80211_chan_req *compat;
        struct ieee80211_chan_req tmp;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!cfg80211_chandef_usable(sdata->local->hw.wiphy,
                                     &chanreq->oper,
                                     IEEE80211_CHAN_DISABLED))
                return -EINVAL;

        /* for non-HT 20 MHz the rest doesn't matter */
        if (chanreq->oper.width == NL80211_CHAN_WIDTH_20_NOHT &&
            cfg80211_chandef_identical(&chanreq->oper, &link_conf->chanreq.oper))
                return 0;

        /* but you cannot switch to/from it */
        if (chanreq->oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            link_conf->chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT)
                return -EINVAL;

        conf = rcu_dereference_protected(link_conf->chanctx_conf,
                                         lockdep_is_held(&local->hw.wiphy->mtx));
        if (!conf)
                return -EINVAL;

        ctx = container_of(conf, struct ieee80211_chanctx, conf);

        compat = ieee80211_chanctx_recheck(local, link, ctx, chanreq, &tmp);
        if (!compat)
                return -EINVAL;

        switch (ctx->replace_state) {
        case IEEE80211_CHANCTX_REPLACE_NONE:
                if (!ieee80211_chanctx_reserved_chanreq(local, ctx, compat,
                                                        &tmp))
                        return -EBUSY;
                break;
        case IEEE80211_CHANCTX_WILL_BE_REPLACED:
                /* TODO: Perhaps the bandwidth change could be treated as a
                 * reservation itself? */
                return -EBUSY;
        case IEEE80211_CHANCTX_REPLACES_OTHER:
                /* channel context that is going to replace another channel
                 * context doesn't really exist and shouldn't be assigned
                 * anywhere yet */
                WARN_ON(1);
                break;
        }

        ieee80211_link_update_chanreq(link, chanreq);

        ieee80211_recalc_chanctx_chantype(local, ctx);

        *changed |= BSS_CHANGED_BANDWIDTH;
        return 0;
}

void ieee80211_link_release_channel(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;

        if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
                return;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        if (rcu_access_pointer(link->conf->chanctx_conf))
                __ieee80211_link_release_channel(link, false);
}

void ieee80211_link_vlan_copy_chanctx(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        unsigned int link_id = link->link_id;
        struct ieee80211_bss_conf *link_conf = link->conf;
        struct ieee80211_bss_conf *ap_conf;
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_sub_if_data *ap;
        struct ieee80211_chanctx_conf *conf;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (WARN_ON(sdata->vif.type != NL80211_IFTYPE_AP_VLAN || !sdata->bss))
                return;

        ap = container_of(sdata->bss, struct ieee80211_sub_if_data, u.ap);

        ap_conf = wiphy_dereference(local->hw.wiphy,
                                    ap->vif.link_conf[link_id]);
        conf = wiphy_dereference(local->hw.wiphy,
                                 ap_conf->chanctx_conf);
        rcu_assign_pointer(link_conf->chanctx_conf, conf);
}

void ieee80211_iter_chan_contexts_atomic(
        struct ieee80211_hw *hw,
        void (*iter)(struct ieee80211_hw *hw,
                     struct ieee80211_chanctx_conf *chanctx_conf,
                     void *data),
        void *iter_data)
{
        struct ieee80211_local *local = hw_to_local(hw);
        struct ieee80211_chanctx *ctx;

        rcu_read_lock();
        list_for_each_entry_rcu(ctx, &local->chanctx_list, list)
                if (ctx->driver_present)
                        iter(hw, &ctx->conf, iter_data);
        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(ieee80211_iter_chan_contexts_atomic);

void ieee80211_iter_chan_contexts_mtx(
        struct ieee80211_hw *hw,
        void (*iter)(struct ieee80211_hw *hw,
                     struct ieee80211_chanctx_conf *chanctx_conf,
                     void *data),
        void *iter_data)
{
        struct ieee80211_local *local = hw_to_local(hw);
        struct ieee80211_chanctx *ctx;

        lockdep_assert_wiphy(hw->wiphy);

        list_for_each_entry(ctx, &local->chanctx_list, list)
                if (ctx->driver_present)
                        iter(hw, &ctx->conf, iter_data);
}
EXPORT_SYMBOL_GPL(ieee80211_iter_chan_contexts_mtx);












































































































   11 


   11 




    1 




   13 




















    3 
    2 






    8 





















    4 
    9 



    9 














    9 
































  256 












    7 


    3 



  248 

  244 
    1 

    4 





  248 




  104 
   31 




    7 



  237 






  197 




   74 


  239 











  230 
    7 










    6 

















   32 
   12 
  202 











    1 






  236 
























    1 




    1 









    1 


    1 


    1 







































































































    2 


    2 
    2 








    2 

    2 























































































































































   10 
  383 


  384 





  360 


  361 
































































   48 
   47 



















    3 





   45 










   12 

   30 








   42 


   42 


    2 



   34 
   33 






   11 

    7 
   17 

    6 
    6 













    3 







    2 
    1 









    4 















   27 












   28 








   28 






   28 

   28 





   11 

   11 
   11 
   11 



   17 








   28 



   28 




   34 












    2 
    2 














    2 


















    2 





































    2 


    2 

    2 



    2 

    2 
    2 








    2 





































































    2 





    2 












































    1 




    1 

    1 







    2 




















   13 







   13 






   13 



   13 




   13 


   13 


   13 

   13 










   12 















   23 





    3 









   78 








   77 

   40 


   38 

    5 

   32 



    6 




   26 






   26 










   28 






   28 

    3 





   23 

    4 

    3 



    9 








   29 






   28 






















   19 













    1 











   19 






   19 








   19 






   19 


   19 








   13 

















































    6 



    8 

    3 







    9 







    1 
    8 







   21 








    1 


   20 


    1 








    5 



   16 



    4 

    4 














    5 

    2 









    8 





    3 




















































    3 
    3 




    2 


    1 




















    3 





















   14 







   14 
   14 




   14 
































   12 

   13 


   12 



    1 































































































































   20 

   14 
    5 






























  326 










  209 


  209 
  209 
   46 
  174 
  172 









  213 



   20 



   20 








  191 


  212 






   11 











    1 








  122 
  121 




    5 


  117 



  114 

    3 




  112 
  100 


   19 
   10 

    6 

    4 























































































































































   73 














   73 















   74 







   86 
   85 


   86 
   86 








    2 





   84 





    1 


   82 







   80 





   78 

   64 





   35 
















    9 

















    9 










    8 

    8 











    1 

    1 
















   34 

    1 













    4 














   21 



   10 





   10 







   21 
   10 





    1 




    2 







   37 



   56 





   12 
    4 
    8 


















    1 







































   59 
   60 

   60 




































   86 










  138 




















  178 





  178 


  178 



  179 

  179 















  147 
   53 




  179 

















   21 







   21 




   21 





   21 



   21 
   21 














   16 










   16 
    3 



















   21 

















   21 




   21 







   21 



   21 
   21 









    9 










    9 

















   13 







   21 




   21 



   13 














   13 




    5 

    4 

    1 



   12 


    4 









   21 



   19 
   16 
    5 







   21 








   21 




   19 






    3 
   16 






    9 










   21 






    5 


   21 
    3 
















































    1 






    3 





    7 




    6 




    5 



















    6 
    5 
























   11 





   10 







    1 



    8 





















































































































































































































































































































































































































































  199 
































  381 




  383 








































































  199 



















  199 





























  199 

































  199 























  199 





















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Implementation of the Transmission Control Protocol(TCP).
 *
 *                IPv4 specific functions
 *
 *                code split from:
 *                linux/ipv4/tcp.c
 *                linux/ipv4/tcp_input.c
 *                linux/ipv4/tcp_output.c
 *
 *                See tcp.c for author information
 */

/*
 * Changes:
 *                David S. Miller        :        New socket lookup architecture.
 *                                        This code is dedicated to John Dyson.
 *                David S. Miller :        Change semantics of established hash,
 *                                        half is devoted to TIME_WAIT sockets
 *                                        and the rest go in the other half.
 *                Andi Kleen :                Add support for syncookies and fixed
 *                                        some bugs: ip options weren't passed to
 *                                        the TCP layer, missed a check for an
 *                                        ACK bit.
 *                Andi Kleen :                Implemented fast path mtu discovery.
 *                                             Fixed many serious bugs in the
 *                                        request_sock handling and moved
 *                                        most of it into the af independent code.
 *                                        Added tail drop and some other bugfixes.
 *                                        Added new listen semantics.
 *                Mike McLagan        :        Routing by source
 *        Juan Jose Ciarlante:                ip_dynaddr bits
 *                Andi Kleen:                various fixes.
 *        Vitaly E. Lavrov        :        Transparent proxy revived after year
 *                                        coma.
 *        Andi Kleen                :        Fix new listen.
 *        Andi Kleen                :        Fix accept error reporting.
 *        YOSHIFUJI Hideaki @USAGI and:        Support IPV6_V6ONLY socket option, which
 *        Alexey Kuznetsov                allow both IPv4 and IPv6 sockets to bind
 *                                        a single port at the same time.
 */

#define pr_fmt(fmt) "TCP: " fmt

#include <linux/bottom_half.h>
#include <linux/types.h>
#include <linux/fcntl.h>
#include <linux/module.h>
#include <linux/random.h>
#include <linux/cache.h>
#include <linux/jhash.h>
#include <linux/init.h>
#include <linux/times.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/sock_diag.h>

#include <net/aligned_data.h>
#include <net/net_namespace.h>
#include <net/icmp.h>
#include <net/inet_hashtables.h>
#include <net/tcp.h>
#include <net/tcp_ecn.h>
#include <net/transp_v6.h>
#include <net/ipv6.h>
#include <net/inet_common.h>
#include <net/inet_ecn.h>
#include <net/timewait_sock.h>
#include <net/xfrm.h>
#include <net/secure_seq.h>
#include <net/busy_poll.h>
#include <net/rstreason.h>
#include <net/psp.h>

#include <linux/inet.h>
#include <linux/ipv6.h>
#include <linux/stddef.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/inetdevice.h>
#include <linux/btf_ids.h>
#include <linux/skbuff_ref.h>

#include <crypto/hash.h>
#include <linux/scatterlist.h>

#include <trace/events/tcp.h>

#ifdef CONFIG_TCP_MD5SIG
static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
                               __be32 daddr, __be32 saddr, const struct tcphdr *th);
#endif

struct inet_hashinfo tcp_hashinfo;

static DEFINE_PER_CPU(struct sock_bh_locked, ipv4_tcp_sk) = {
        .bh_lock = INIT_LOCAL_LOCK(bh_lock),
};

static DEFINE_MUTEX(tcp_exit_batch_mutex);

static u32 tcp_v4_init_seq(const struct sk_buff *skb)
{
        return secure_tcp_seq(ip_hdr(skb)->daddr,
                              ip_hdr(skb)->saddr,
                              tcp_hdr(skb)->dest,
                              tcp_hdr(skb)->source);
}

static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb)
{
        return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr);
}

int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
{
        int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
        const struct inet_timewait_sock *tw = inet_twsk(sktw);
        const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
        struct tcp_sock *tp = tcp_sk(sk);
        int ts_recent_stamp;
        u32 reuse_thresh;

        if (READ_ONCE(tw->tw_substate) == TCP_FIN_WAIT2)
                reuse = 0;

        if (reuse == 2) {
                /* Still does not detect *everything* that goes through
                 * lo, since we require a loopback src or dst address
                 * or direct binding to 'lo' interface.
                 */
                bool loopback = false;
                if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
                        loopback = true;
#if IS_ENABLED(CONFIG_IPV6)
                if (tw->tw_family == AF_INET6) {
                        if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
                            ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
                            ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
                            ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
                                loopback = true;
                } else
#endif
                {
                        if (ipv4_is_loopback(tw->tw_daddr) ||
                            ipv4_is_loopback(tw->tw_rcv_saddr))
                                loopback = true;
                }
                if (!loopback)
                        reuse = 0;
        }

        /* With PAWS, it is safe from the viewpoint
           of data integrity. Even without PAWS it is safe provided sequence
           spaces do not overlap i.e. at data rates <= 80Mbit/sec.

           Actually, the idea is close to VJ's one, only timestamp cache is
           held not per host, but per port pair and TW bucket is used as state
           holder.

           If TW bucket has been already destroyed we fall back to VJ's scheme
           and use initial timestamp retrieved from peer table.
         */
        ts_recent_stamp = READ_ONCE(tcptw->tw_ts_recent_stamp);
        reuse_thresh = READ_ONCE(tw->tw_entry_stamp) +
                       READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse_delay);
        if (ts_recent_stamp &&
            (!twp || (reuse && time_after32(tcp_clock_ms(), reuse_thresh)))) {
                /* inet_twsk_hashdance_schedule() sets sk_refcnt after putting twsk
                 * and releasing the bucket lock.
                 */
                if (unlikely(!refcount_inc_not_zero(&sktw->sk_refcnt)))
                        return 0;

                /* In case of repair and re-using TIME-WAIT sockets we still
                 * want to be sure that it is safe as above but honor the
                 * sequence numbers and time stamps set as part of the repair
                 * process.
                 *
                 * Without this check re-using a TIME-WAIT socket with TCP
                 * repair would accumulate a -1 on the repair assigned
                 * sequence number. The first time it is reused the sequence
                 * is -1, the second time -2, etc. This fixes that issue
                 * without appearing to create any others.
                 */
                if (likely(!tp->repair)) {
                        u32 seq = tcptw->tw_snd_nxt + 65535 + 2;

                        if (!seq)
                                seq = 1;
                        WRITE_ONCE(tp->write_seq, seq);
                        tp->rx_opt.ts_recent           = READ_ONCE(tcptw->tw_ts_recent);
                        tp->rx_opt.ts_recent_stamp = ts_recent_stamp;
                }

                return 1;
        }

        return 0;
}
EXPORT_IPV6_MOD_GPL(tcp_twsk_unique);

static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
                              int addr_len)
{
        /* This check is replicated from tcp_v4_connect() and intended to
         * prevent BPF program called below from accessing bytes that are out
         * of the bound specified by user in addr_len.
         */
        if (addr_len < sizeof(struct sockaddr_in))
                return -EINVAL;

        sock_owned_by_me(sk);

        return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr, &addr_len);
}

/* This will initiate an outgoing connection. */
int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
{
        struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
        struct inet_timewait_death_row *tcp_death_row;
        struct inet_sock *inet = inet_sk(sk);
        struct tcp_sock *tp = tcp_sk(sk);
        struct ip_options_rcu *inet_opt;
        struct net *net = sock_net(sk);
        __be16 orig_sport, orig_dport;
        __be32 daddr, nexthop;
        struct flowi4 *fl4;
        struct rtable *rt;
        int err;

        if (addr_len < sizeof(struct sockaddr_in))
                return -EINVAL;

        if (usin->sin_family != AF_INET)
                return -EAFNOSUPPORT;

        nexthop = daddr = usin->sin_addr.s_addr;
        inet_opt = rcu_dereference_protected(inet->inet_opt,
                                             lockdep_sock_is_held(sk));
        if (inet_opt && inet_opt->opt.srr) {
                if (!daddr)
                        return -EINVAL;
                nexthop = inet_opt->opt.faddr;
        }

        orig_sport = inet->inet_sport;
        orig_dport = usin->sin_port;
        fl4 = &inet->cork.fl.u.ip4;
        rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
                              sk->sk_bound_dev_if, IPPROTO_TCP, orig_sport,
                              orig_dport, sk);
        if (IS_ERR(rt)) {
                err = PTR_ERR(rt);
                if (err == -ENETUNREACH)
                        IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
                return err;
        }

        if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
                ip_rt_put(rt);
                return -ENETUNREACH;
        }

        if (!inet_opt || !inet_opt->opt.srr)
                daddr = fl4->daddr;

        tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;

        if (!inet->inet_saddr) {
                err = inet_bhash2_update_saddr(sk,  &fl4->saddr, AF_INET);
                if (err) {
                        ip_rt_put(rt);
                        return err;
                }
        } else {
                sk_rcv_saddr_set(sk, inet->inet_saddr);
        }

        if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
                /* Reset inherited state */
                tp->rx_opt.ts_recent           = 0;
                tp->rx_opt.ts_recent_stamp = 0;
                if (likely(!tp->repair))
                        WRITE_ONCE(tp->write_seq, 0);
        }

        inet->inet_dport = usin->sin_port;
        sk_daddr_set(sk, daddr);

        inet_csk(sk)->icsk_ext_hdr_len = psp_sk_overhead(sk);
        if (inet_opt)
                inet_csk(sk)->icsk_ext_hdr_len += inet_opt->opt.optlen;

        tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;

        /* Socket identity is still unknown (sport may be zero).
         * However we set state to SYN-SENT and not releasing socket
         * lock select source port, enter ourselves into the hash tables and
         * complete initialization after this.
         */
        tcp_set_state(sk, TCP_SYN_SENT);
        err = inet_hash_connect(tcp_death_row, sk);
        if (err)
                goto failure;

        sk_set_txhash(sk);

        rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
                               inet->inet_sport, inet->inet_dport, sk);
        if (IS_ERR(rt)) {
                err = PTR_ERR(rt);
                rt = NULL;
                goto failure;
        }
        tp->tcp_usec_ts = dst_tcp_usec_ts(&rt->dst);
        /* OK, now commit destination to socket.  */
        sk->sk_gso_type = SKB_GSO_TCPV4;
        sk_setup_caps(sk, &rt->dst);
        rt = NULL;

        if (likely(!tp->repair)) {
                if (!tp->write_seq)
                        WRITE_ONCE(tp->write_seq,
                                   secure_tcp_seq(inet->inet_saddr,
                                                  inet->inet_daddr,
                                                  inet->inet_sport,
                                                  usin->sin_port));
                WRITE_ONCE(tp->tsoffset,
                           secure_tcp_ts_off(net, inet->inet_saddr,
                                             inet->inet_daddr));
        }

        atomic_set(&inet->inet_id, get_random_u16());

        if (tcp_fastopen_defer_connect(sk, &err))
                return err;
        if (err)
                goto failure;

        err = tcp_connect(sk);

        if (err)
                goto failure;

        return 0;

failure:
        /*
         * This unhashes the socket and releases the local port,
         * if necessary.
         */
        tcp_set_state(sk, TCP_CLOSE);
        inet_bhash2_reset_saddr(sk);
        ip_rt_put(rt);
        sk->sk_route_caps = 0;
        inet->inet_dport = 0;
        return err;
}
EXPORT_IPV6_MOD(tcp_v4_connect);

/*
 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
 * It can be called through tcp_release_cb() if socket was owned by user
 * at the time tcp_v4_err() was called to handle ICMP message.
 */
void tcp_v4_mtu_reduced(struct sock *sk)
{
        struct inet_sock *inet = inet_sk(sk);
        struct dst_entry *dst;
        u32 mtu;

        if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
                return;
        mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
        dst = inet_csk_update_pmtu(sk, mtu);
        if (!dst)
                return;

        /* Something is about to be wrong... Remember soft error
         * for the case, if this connection will not able to recover.
         */
        if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
                WRITE_ONCE(sk->sk_err_soft, EMSGSIZE);

        mtu = dst_mtu(dst);

        if (inet->pmtudisc != IP_PMTUDISC_DONT &&
            ip_sk_accept_pmtu(sk) &&
            inet_csk(sk)->icsk_pmtu_cookie > mtu) {
                tcp_sync_mss(sk, mtu);

                /* Resend the TCP packet because it's
                 * clear that the old packet has been
                 * dropped. This is the new "fast" path mtu
                 * discovery.
                 */
                tcp_simple_retransmit(sk);
        } /* else let the usual retransmit timer handle it */
}
EXPORT_IPV6_MOD(tcp_v4_mtu_reduced);

static void do_redirect(struct sk_buff *skb, struct sock *sk)
{
        struct dst_entry *dst = __sk_dst_check(sk, 0);

        if (dst)
                dst->ops->redirect(dst, sk, skb);
}


/* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
void tcp_req_err(struct sock *sk, u32 seq, bool abort)
{
        struct request_sock *req = inet_reqsk(sk);
        struct net *net = sock_net(sk);

        /* ICMPs are not backlogged, hence we cannot get
         * an established socket here.
         */
        if (seq != tcp_rsk(req)->snt_isn) {
                __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
        } else if (abort) {
                /*
                 * Still in SYN_RECV, just remove it silently.
                 * There is no good way to pass the error to the newly
                 * created socket, and POSIX does not want network
                 * errors returned from accept().
                 */
                inet_csk_reqsk_queue_drop(req->rsk_listener, req);
                tcp_listendrop(req->rsk_listener);
        }
        reqsk_put(req);
}
EXPORT_IPV6_MOD(tcp_req_err);

/* TCP-LD (RFC 6069) logic */
void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
{
        struct inet_connection_sock *icsk = inet_csk(sk);
        struct tcp_sock *tp = tcp_sk(sk);
        struct sk_buff *skb;
        s32 remaining;
        u32 delta_us;

        if (sock_owned_by_user(sk))
                return;

        if (seq != tp->snd_una  || !icsk->icsk_retransmits ||
            !icsk->icsk_backoff)
                return;

        skb = tcp_rtx_queue_head(sk);
        if (WARN_ON_ONCE(!skb))
                return;

        icsk->icsk_backoff--;
        icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
        icsk->icsk_rto = inet_csk_rto_backoff(icsk, tcp_rto_max(sk));

        tcp_mstamp_refresh(tp);
        delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
        remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);

        if (remaining > 0) {
                tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, remaining, false);
        } else {
                /* RTO revert clocked out retransmission.
                 * Will retransmit now.
                 */
                tcp_retransmit_timer(sk);
        }
}
EXPORT_IPV6_MOD(tcp_ld_RTO_revert);

/*
 * This routine is called by the ICMP module when it gets some
 * sort of error condition.  If err < 0 then the socket should
 * be closed and the error returned to the user.  If err > 0
 * it's just the icmp type << 8 | icmp code.  After adjustment
 * header points to the first 8 bytes of the tcp header.  We need
 * to find the appropriate port.
 *
 * The locking strategy used here is very "optimistic". When
 * someone else accesses the socket the ICMP is just dropped
 * and for some paths there is no check at all.
 * A more general error queue to queue errors for later handling
 * is probably better.
 *
 */

int tcp_v4_err(struct sk_buff *skb, u32 info)
{
        const struct iphdr *iph = (const struct iphdr *)skb->data;
        struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
        struct net *net = dev_net_rcu(skb->dev);
        const int type = icmp_hdr(skb)->type;
        const int code = icmp_hdr(skb)->code;
        struct request_sock *fastopen;
        struct tcp_sock *tp;
        u32 seq, snd_una;
        struct sock *sk;
        int err;

        sk = __inet_lookup_established(net, iph->daddr, th->dest, iph->saddr,
                                       ntohs(th->source), inet_iif(skb), 0);
        if (!sk) {
                __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
                return -ENOENT;
        }
        if (sk->sk_state == TCP_TIME_WAIT) {
                /* To increase the counter of ignored icmps for TCP-AO */
                tcp_ao_ignore_icmp(sk, AF_INET, type, code);
                inet_twsk_put(inet_twsk(sk));
                return 0;
        }
        seq = ntohl(th->seq);
        if (sk->sk_state == TCP_NEW_SYN_RECV) {
                tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
                                     type == ICMP_TIME_EXCEEDED ||
                                     (type == ICMP_DEST_UNREACH &&
                                      (code == ICMP_NET_UNREACH ||
                                       code == ICMP_HOST_UNREACH)));
                return 0;
        }

        if (tcp_ao_ignore_icmp(sk, AF_INET, type, code)) {
                sock_put(sk);
                return 0;
        }

        bh_lock_sock(sk);
        /* If too many ICMPs get dropped on busy
         * servers this needs to be solved differently.
         * We do take care of PMTU discovery (RFC1191) special case :
         * we can receive locally generated ICMP messages while socket is held.
         */
        if (sock_owned_by_user(sk)) {
                if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
                        __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
        }
        if (sk->sk_state == TCP_CLOSE)
                goto out;

        if (static_branch_unlikely(&ip4_min_ttl)) {
                /* min_ttl can be changed concurrently from do_ip_setsockopt() */
                if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
                        __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
                        goto out;
                }
        }

        tp = tcp_sk(sk);
        /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
        fastopen = rcu_dereference(tp->fastopen_rsk);
        snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
        if (sk->sk_state != TCP_LISTEN &&
            !between(seq, snd_una, tp->snd_nxt)) {
                __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
                goto out;
        }

        switch (type) {
        case ICMP_REDIRECT:
                if (!sock_owned_by_user(sk))
                        do_redirect(skb, sk);
                goto out;
        case ICMP_SOURCE_QUENCH:
                /* Just silently ignore these. */
                goto out;
        case ICMP_PARAMETERPROB:
                err = EPROTO;
                break;
        case ICMP_DEST_UNREACH:
                if (code > NR_ICMP_UNREACH)
                        goto out;

                if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
                        /* We are not interested in TCP_LISTEN and open_requests
                         * (SYN-ACKs send out by Linux are always <576bytes so
                         * they should go through unfragmented).
                         */
                        if (sk->sk_state == TCP_LISTEN)
                                goto out;

                        WRITE_ONCE(tp->mtu_info, info);
                        if (!sock_owned_by_user(sk)) {
                                tcp_v4_mtu_reduced(sk);
                        } else {
                                if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
                                        sock_hold(sk);
                        }
                        goto out;
                }

                err = icmp_err_convert[code].errno;
                /* check if this ICMP message allows revert of backoff.
                 * (see RFC 6069)
                 */
                if (!fastopen &&
                    (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
                        tcp_ld_RTO_revert(sk, seq);
                break;
        case ICMP_TIME_EXCEEDED:
                err = EHOSTUNREACH;
                break;
        default:
                goto out;
        }

        switch (sk->sk_state) {
        case TCP_SYN_SENT:
        case TCP_SYN_RECV:
                /* Only in fast or simultaneous open. If a fast open socket is
                 * already accepted it is treated as a connected one below.
                 */
                if (fastopen && !fastopen->sk)
                        break;

                ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);

                if (!sock_owned_by_user(sk))
                        tcp_done_with_error(sk, err);
                else
                        WRITE_ONCE(sk->sk_err_soft, err);
                goto out;
        }

        /* If we've already connected we will keep trying
         * until we time out, or the user gives up.
         *
         * rfc1122 4.2.3.9 allows to consider as hard errors
         * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
         * but it is obsoleted by pmtu discovery).
         *
         * Note, that in modern internet, where routing is unreliable
         * and in each dark corner broken firewalls sit, sending random
         * errors ordered by their masters even this two messages finally lose
         * their original sense (even Linux sends invalid PORT_UNREACHs)
         *
         * Now we are in compliance with RFCs.
         *                                                        --ANK (980905)
         */

        if (!sock_owned_by_user(sk) &&
            inet_test_bit(RECVERR, sk)) {
                WRITE_ONCE(sk->sk_err, err);
                sk_error_report(sk);
        } else        { /* Only an error on timeout */
                WRITE_ONCE(sk->sk_err_soft, err);
        }

out:
        bh_unlock_sock(sk);
        sock_put(sk);
        return 0;
}

void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
{
        struct tcphdr *th = tcp_hdr(skb);

        th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
        skb->csum_start = skb_transport_header(skb) - skb->head;
        skb->csum_offset = offsetof(struct tcphdr, check);
}

/* This routine computes an IPv4 TCP checksum. */
void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
{
        const struct inet_sock *inet = inet_sk(sk);

        __tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
}
EXPORT_IPV6_MOD(tcp_v4_send_check);

#define REPLY_OPTIONS_LEN      (MAX_TCP_OPTION_SPACE / sizeof(__be32))

static bool tcp_v4_ao_sign_reset(const struct sock *sk, struct sk_buff *skb,
                                 const struct tcp_ao_hdr *aoh,
                                 struct ip_reply_arg *arg, struct tcphdr *reply,
                                 __be32 reply_options[REPLY_OPTIONS_LEN])
{
#ifdef CONFIG_TCP_AO
        int sdif = tcp_v4_sdif(skb);
        int dif = inet_iif(skb);
        int l3index = sdif ? dif : 0;
        bool allocated_traffic_key;
        struct tcp_ao_key *key;
        char *traffic_key;
        bool drop = true;
        u32 ao_sne = 0;
        u8 keyid;

        rcu_read_lock();
        if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, ntohl(reply->seq),
                                 &key, &traffic_key, &allocated_traffic_key,
                                 &keyid, &ao_sne))
                goto out;

        reply_options[0] = htonl((TCPOPT_AO << 24) | (tcp_ao_len(key) << 16) |
                                 (aoh->rnext_keyid << 8) | keyid);
        arg->iov[0].iov_len += tcp_ao_len_aligned(key);
        reply->doff = arg->iov[0].iov_len / 4;

        if (tcp_ao_hash_hdr(AF_INET, (char *)&reply_options[1],
                            key, traffic_key,
                            (union tcp_ao_addr *)&ip_hdr(skb)->saddr,
                            (union tcp_ao_addr *)&ip_hdr(skb)->daddr,
                            reply, ao_sne))
                goto out;
        drop = false;
out:
        rcu_read_unlock();
        if (allocated_traffic_key)
                kfree(traffic_key);
        return drop;
#else
        return true;
#endif
}

/*
 *        This routine will send an RST to the other tcp.
 *
 *        Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
 *                      for reset.
 *        Answer: if a packet caused RST, it is not for a socket
 *                existing in our system, if it is matched to a socket,
 *                it is just duplicate segment or bug in other side's TCP.
 *                So that we build reply only basing on parameters
 *                arrived with segment.
 *        Exception: precedence violation. We do not implement it in any case.
 */

static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb,
                              enum sk_rst_reason reason)
{
        const struct tcphdr *th = tcp_hdr(skb);
        struct {
                struct tcphdr th;
                __be32 opt[REPLY_OPTIONS_LEN];
        } rep;
        const __u8 *md5_hash_location = NULL;
        const struct tcp_ao_hdr *aoh;
        struct ip_reply_arg arg;
#ifdef CONFIG_TCP_MD5SIG
        struct tcp_md5sig_key *key = NULL;
        unsigned char newhash[16];
        struct sock *sk1 = NULL;
        int genhash;
#endif
        u64 transmit_time = 0;
        struct sock *ctl_sk;
        struct net *net;
        u32 txhash = 0;

        /* Never send a reset in response to a reset. */
        if (th->rst)
                return;

        /* If sk not NULL, it means we did a successful lookup and incoming
         * route had to be correct. prequeue might have dropped our dst.
         */
        if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
                return;

        /* Swap the send and the receive. */
        memset(&rep, 0, sizeof(rep));
        rep.th.dest   = th->source;
        rep.th.source = th->dest;
        rep.th.doff   = sizeof(struct tcphdr) / 4;
        rep.th.rst    = 1;

        if (th->ack) {
                rep.th.seq = th->ack_seq;
        } else {
                rep.th.ack = 1;
                rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
                                       skb->len - (th->doff << 2));
        }

        memset(&arg, 0, sizeof(arg));
        arg.iov[0].iov_base = (unsigned char *)&rep;
        arg.iov[0].iov_len  = sizeof(rep.th);

        net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb);

        /* Invalid TCP option size or twice included auth */
        if (tcp_parse_auth_options(tcp_hdr(skb), &md5_hash_location, &aoh))
                return;

        if (aoh && tcp_v4_ao_sign_reset(sk, skb, aoh, &arg, &rep.th, rep.opt))
                return;

#ifdef CONFIG_TCP_MD5SIG
        rcu_read_lock();
        if (sk && sk_fullsock(sk)) {
                const union tcp_md5_addr *addr;
                int l3index;

                /* sdif set, means packet ingressed via a device
                 * in an L3 domain and inet_iif is set to it.
                 */
                l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
                addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
                key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
        } else if (md5_hash_location) {
                const union tcp_md5_addr *addr;
                int sdif = tcp_v4_sdif(skb);
                int dif = inet_iif(skb);
                int l3index;

                /*
                 * active side is lost. Try to find listening socket through
                 * source port, and then find md5 key through listening socket.
                 * we are not loose security here:
                 * Incoming packet is checked with md5 hash with finding key,
                 * no RST generated if md5 hash doesn't match.
                 */
                sk1 = __inet_lookup_listener(net, NULL, 0, ip_hdr(skb)->saddr,
                                             th->source, ip_hdr(skb)->daddr,
                                             ntohs(th->source), dif, sdif);
                /* don't send rst if it can't find key */
                if (!sk1)
                        goto out;

                /* sdif set, means packet ingressed via a device
                 * in an L3 domain and dif is set to it.
                 */
                l3index = sdif ? dif : 0;
                addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
                key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
                if (!key)
                        goto out;


                genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
                if (genhash || memcmp(md5_hash_location, newhash, 16) != 0)
                        goto out;

        }

        if (key) {
                rep.opt[0] = htonl((TCPOPT_NOP << 24) |
                                   (TCPOPT_NOP << 16) |
                                   (TCPOPT_MD5SIG << 8) |
                                   TCPOLEN_MD5SIG);
                /* Update length and the length the header thinks exists */
                arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
                rep.th.doff = arg.iov[0].iov_len / 4;

                tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
                                     key, ip_hdr(skb)->saddr,
                                     ip_hdr(skb)->daddr, &rep.th);
        }
#endif
        /* Can't co-exist with TCPMD5, hence check rep.opt[0] */
        if (rep.opt[0] == 0) {
                __be32 mrst = mptcp_reset_option(skb);

                if (mrst) {
                        rep.opt[0] = mrst;
                        arg.iov[0].iov_len += sizeof(mrst);
                        rep.th.doff = arg.iov[0].iov_len / 4;
                }
        }

        arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
                                      ip_hdr(skb)->saddr, /* XXX */
                                      arg.iov[0].iov_len, IPPROTO_TCP, 0);
        arg.csumoffset = offsetof(struct tcphdr, check) / 2;
        arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;

        /* When socket is gone, all binding information is lost.
         * routing might fail in this case. No choice here, if we choose to force
         * input interface, we will misroute in case of asymmetric route.
         */
        if (sk)
                arg.bound_dev_if = sk->sk_bound_dev_if;

        trace_tcp_send_reset(sk, skb, reason);

        BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
                     offsetof(struct inet_timewait_sock, tw_bound_dev_if));

        /* ECN bits of TW reset are cleared */
        arg.tos = ip_hdr(skb)->tos & ~INET_ECN_MASK;
        arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
        local_bh_disable();
        local_lock_nested_bh(&ipv4_tcp_sk.bh_lock);
        ctl_sk = this_cpu_read(ipv4_tcp_sk.sock);

        sock_net_set(ctl_sk, net);
        if (sk) {
                ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
                                   inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark);
                ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
                                   inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority);
                transmit_time = tcp_transmit_time(sk);
                xfrm_sk_clone_policy(ctl_sk, sk);
                txhash = (sk->sk_state == TCP_TIME_WAIT) ?
                         inet_twsk(sk)->tw_txhash : sk->sk_txhash;
        } else {
                ctl_sk->sk_mark = 0;
                ctl_sk->sk_priority = 0;
        }
        ip_send_unicast_reply(ctl_sk, sk,
                              skb, &TCP_SKB_CB(skb)->header.h4.opt,
                              ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
                              &arg, arg.iov[0].iov_len,
                              transmit_time, txhash);

        xfrm_sk_free_policy(ctl_sk);
        sock_net_set(ctl_sk, &init_net);
        __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
        __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
        local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock);
        local_bh_enable();

#ifdef CONFIG_TCP_MD5SIG
out:
        rcu_read_unlock();
#endif
}

/* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
   outside socket context is ugly, certainly. What can I do?
 */

static void tcp_v4_send_ack(const struct sock *sk,
                            struct sk_buff *skb, u32 seq, u32 ack,
                            u32 win, u32 tsval, u32 tsecr, int oif,
                            struct tcp_key *key,
                            int reply_flags, u8 tos, u32 txhash)
{
        const struct tcphdr *th = tcp_hdr(skb);
        struct {
                struct tcphdr th;
                __be32 opt[(MAX_TCP_OPTION_SPACE  >> 2)];
        } rep;
        struct net *net = sock_net(sk);
        struct ip_reply_arg arg;
        struct sock *ctl_sk;
        u64 transmit_time;

        memset(&rep.th, 0, sizeof(struct tcphdr));
        memset(&arg, 0, sizeof(arg));

        arg.iov[0].iov_base = (unsigned char *)&rep;
        arg.iov[0].iov_len  = sizeof(rep.th);
        if (tsecr) {
                rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
                                   (TCPOPT_TIMESTAMP << 8) |
                                   TCPOLEN_TIMESTAMP);
                rep.opt[1] = htonl(tsval);
                rep.opt[2] = htonl(tsecr);
                arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
        }

        /* Swap the send and the receive. */
        rep.th.dest    = th->source;
        rep.th.source  = th->dest;
        rep.th.doff    = arg.iov[0].iov_len / 4;
        rep.th.seq     = htonl(seq);
        rep.th.ack_seq = htonl(ack);
        rep.th.ack     = 1;
        rep.th.window  = htons(win);

#ifdef CONFIG_TCP_MD5SIG
        if (tcp_key_is_md5(key)) {
                int offset = (tsecr) ? 3 : 0;

                rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
                                          (TCPOPT_NOP << 16) |
                                          (TCPOPT_MD5SIG << 8) |
                                          TCPOLEN_MD5SIG);
                arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
                rep.th.doff = arg.iov[0].iov_len/4;

                tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
                                    key->md5_key, ip_hdr(skb)->saddr,
                                    ip_hdr(skb)->daddr, &rep.th);
        }
#endif
#ifdef CONFIG_TCP_AO
        if (tcp_key_is_ao(key)) {
                int offset = (tsecr) ? 3 : 0;

                rep.opt[offset++] = htonl((TCPOPT_AO << 24) |
                                          (tcp_ao_len(key->ao_key) << 16) |
                                          (key->ao_key->sndid << 8) |
                                          key->rcv_next);
                arg.iov[0].iov_len += tcp_ao_len_aligned(key->ao_key);
                rep.th.doff = arg.iov[0].iov_len / 4;

                tcp_ao_hash_hdr(AF_INET, (char *)&rep.opt[offset],
                                key->ao_key, key->traffic_key,
                                (union tcp_ao_addr *)&ip_hdr(skb)->saddr,
                                (union tcp_ao_addr *)&ip_hdr(skb)->daddr,
                                &rep.th, key->sne);
        }
#endif
        arg.flags = reply_flags;
        arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
                                      ip_hdr(skb)->saddr, /* XXX */
                                      arg.iov[0].iov_len, IPPROTO_TCP, 0);
        arg.csumoffset = offsetof(struct tcphdr, check) / 2;
        if (oif)
                arg.bound_dev_if = oif;
        arg.tos = tos;
        arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
        local_bh_disable();
        local_lock_nested_bh(&ipv4_tcp_sk.bh_lock);
        ctl_sk = this_cpu_read(ipv4_tcp_sk.sock);
        sock_net_set(ctl_sk, net);
        ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
                           inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark);
        ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
                           inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority);
        transmit_time = tcp_transmit_time(sk);
        ip_send_unicast_reply(ctl_sk, sk,
                              skb, &TCP_SKB_CB(skb)->header.h4.opt,
                              ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
                              &arg, arg.iov[0].iov_len,
                              transmit_time, txhash);

        sock_net_set(ctl_sk, &init_net);
        __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
        local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock);
        local_bh_enable();
}

static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb,
                                enum tcp_tw_status tw_status)
{
        struct inet_timewait_sock *tw = inet_twsk(sk);
        struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
        struct tcp_key key = {};
        u8 tos = tw->tw_tos;

        /* Cleaning only ECN bits of TW ACKs of oow data or is paws_reject,
         * while not cleaning ECN bits of other TW ACKs to avoid these ACKs
         * being placed in a different service queues (Classic rather than L4S)
         */
        if (tw_status == TCP_TW_ACK_OOW)
                tos &= ~INET_ECN_MASK;

#ifdef CONFIG_TCP_AO
        struct tcp_ao_info *ao_info;

        if (static_branch_unlikely(&tcp_ao_needed.key)) {
                /* FIXME: the segment to-be-acked is not verified yet */
                ao_info = rcu_dereference(tcptw->ao_info);
                if (ao_info) {
                        const struct tcp_ao_hdr *aoh;

                        if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) {
                                inet_twsk_put(tw);
                                return;
                        }

                        if (aoh)
                                key.ao_key = tcp_ao_established_key(sk, ao_info,
                                                                    aoh->rnext_keyid, -1);
                }
        }
        if (key.ao_key) {
                struct tcp_ao_key *rnext_key;

                key.traffic_key = snd_other_key(key.ao_key);
                key.sne = READ_ONCE(ao_info->snd_sne);
                rnext_key = READ_ONCE(ao_info->rnext_key);
                key.rcv_next = rnext_key->rcvid;
                key.type = TCP_KEY_AO;
#else
        if (0) {
#endif
        } else if (static_branch_tcp_md5()) {
                key.md5_key = tcp_twsk_md5_key(tcptw);
                if (key.md5_key)
                        key.type = TCP_KEY_MD5;
        }

        tcp_v4_send_ack(sk, skb,
                        tcptw->tw_snd_nxt, READ_ONCE(tcptw->tw_rcv_nxt),
                        tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
                        tcp_tw_tsval(tcptw),
                        READ_ONCE(tcptw->tw_ts_recent),
                        tw->tw_bound_dev_if, &key,
                        tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
                        tos,
                        tw->tw_txhash);

        inet_twsk_put(tw);
}

static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
                                  struct request_sock *req)
{
        struct tcp_key key = {};

        /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
         * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
         */
        u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
                                             tcp_sk(sk)->snd_nxt;

#ifdef CONFIG_TCP_AO
        if (static_branch_unlikely(&tcp_ao_needed.key) &&
            tcp_rsk_used_ao(req)) {
                const union tcp_md5_addr *addr;
                const struct tcp_ao_hdr *aoh;
                int l3index;

                /* Invalid TCP option size or twice included auth */
                if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh))
                        return;
                if (!aoh)
                        return;

                addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
                l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
                key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET,
                                              aoh->rnext_keyid, -1);
                if (unlikely(!key.ao_key)) {
                        /* Send ACK with any matching MKT for the peer */
                        key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET, -1, -1);
                        /* Matching key disappeared (user removed the key?)
                         * let the handshake timeout.
                         */
                        if (!key.ao_key) {
                                net_info_ratelimited("TCP-AO key for (%pI4, %d)->(%pI4, %d) suddenly disappeared, won't ACK new connection\n",
                                                     addr,
                                                     ntohs(tcp_hdr(skb)->source),
                                                     &ip_hdr(skb)->daddr,
                                                     ntohs(tcp_hdr(skb)->dest));
                                return;
                        }
                }
                key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC);
                if (!key.traffic_key)
                        return;

                key.type = TCP_KEY_AO;
                key.rcv_next = aoh->keyid;
                tcp_v4_ao_calc_key_rsk(key.ao_key, key.traffic_key, req);
#else
        if (0) {
#endif
        } else if (static_branch_tcp_md5()) {
                const union tcp_md5_addr *addr;
                int l3index;

                addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
                l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
                key.md5_key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
                if (key.md5_key)
                        key.type = TCP_KEY_MD5;
        }

        /* Cleaning ECN bits of TW ACKs of oow data or is paws_reject */
        tcp_v4_send_ack(sk, skb, seq,
                        tcp_rsk(req)->rcv_nxt,
                        tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale,
                        tcp_rsk_tsval(tcp_rsk(req)),
                        req->ts_recent,
                        0, &key,
                        inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
                        ip_hdr(skb)->tos & ~INET_ECN_MASK,
                        READ_ONCE(tcp_rsk(req)->txhash));
        if (tcp_key_is_ao(&key))
                kfree(key.traffic_key);
}

/*
 *        Send a SYN-ACK after having received a SYN.
 *        This still operates on a request_sock only, not on a big
 *        socket.
 */
static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
                              struct flowi *fl,
                              struct request_sock *req,
                              struct tcp_fastopen_cookie *foc,
                              enum tcp_synack_type synack_type,
                              struct sk_buff *syn_skb)
{
        struct inet_request_sock *ireq = inet_rsk(req);
        struct flowi4 fl4;
        int err = -1;
        struct sk_buff *skb;
        u8 tos;

        /* First, grab a route. */
        if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
                return -1;

        skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);

        if (skb) {
                tcp_rsk(req)->syn_ect_snt = inet_sk(sk)->tos & INET_ECN_MASK;
                __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);

                tos = READ_ONCE(inet_sk(sk)->tos);

                if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
                        tos = (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
                              (tos & INET_ECN_MASK);

                if (!INET_ECN_is_capable(tos) &&
                    tcp_bpf_ca_needs_ecn((struct sock *)req))
                        tos |= INET_ECN_ECT_0;

                rcu_read_lock();
                err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
                                            ireq->ir_rmt_addr,
                                            rcu_dereference(ireq->ireq_opt),
                                            tos);
                rcu_read_unlock();
                err = net_xmit_eval(err);
        }

        return err;
}

/*
 *        IPv4 request_sock destructor.
 */
static void tcp_v4_reqsk_destructor(struct request_sock *req)
{
        kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
}

#ifdef CONFIG_TCP_MD5SIG
/*
 * RFC2385 MD5 checksumming requires a mapping of
 * IP address->MD5 Key.
 * We need to maintain these in the sk structure.
 */

DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_md5_needed, HZ);
EXPORT_IPV6_MOD(tcp_md5_needed);

static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig_key *new)
{
        if (!old)
                return true;

        /* l3index always overrides non-l3index */
        if (old->l3index && new->l3index == 0)
                return false;
        if (old->l3index == 0 && new->l3index)
                return true;

        return old->prefixlen < new->prefixlen;
}

/* Find the Key structure for an address.  */
struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
                                           const union tcp_md5_addr *addr,
                                           int family, bool any_l3index)
{
        const struct tcp_sock *tp = tcp_sk(sk);
        struct tcp_md5sig_key *key;
        const struct tcp_md5sig_info *md5sig;
        __be32 mask;
        struct tcp_md5sig_key *best_match = NULL;
        bool match;

        /* caller either holds rcu_read_lock() or socket lock */
        md5sig = rcu_dereference_check(tp->md5sig_info,
                                       lockdep_sock_is_held(sk));
        if (!md5sig)
                return NULL;

        hlist_for_each_entry_rcu(key, &md5sig->head, node,
                                 lockdep_sock_is_held(sk)) {
                if (key->family != family)
                        continue;
                if (!any_l3index && key->flags & TCP_MD5SIG_FLAG_IFINDEX &&
                    key->l3index != l3index)
                        continue;
                if (family == AF_INET) {
                        mask = inet_make_mask(key->prefixlen);
                        match = (key->addr.a4.s_addr & mask) ==
                                (addr->a4.s_addr & mask);
#if IS_ENABLED(CONFIG_IPV6)
                } else if (family == AF_INET6) {
                        match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
                                                  key->prefixlen);
#endif
                } else {
                        match = false;
                }

                if (match && better_md5_match(best_match, key))
                        best_match = key;
        }
        return best_match;
}
EXPORT_IPV6_MOD(__tcp_md5_do_lookup);

static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
                                                      const union tcp_md5_addr *addr,
                                                      int family, u8 prefixlen,
                                                      int l3index, u8 flags)
{
        const struct tcp_sock *tp = tcp_sk(sk);
        struct tcp_md5sig_key *key;
        unsigned int size = sizeof(struct in_addr);
        const struct tcp_md5sig_info *md5sig;

        /* caller either holds rcu_read_lock() or socket lock */
        md5sig = rcu_dereference_check(tp->md5sig_info,
                                       lockdep_sock_is_held(sk));
        if (!md5sig)
                return NULL;
#if IS_ENABLED(CONFIG_IPV6)
        if (family == AF_INET6)
                size = sizeof(struct in6_addr);
#endif
        hlist_for_each_entry_rcu(key, &md5sig->head, node,
                                 lockdep_sock_is_held(sk)) {
                if (key->family != family)
                        continue;
                if ((key->flags & TCP_MD5SIG_FLAG_IFINDEX) != (flags & TCP_MD5SIG_FLAG_IFINDEX))
                        continue;
                if (key->l3index != l3index)
                        continue;
                if (!memcmp(&key->addr, addr, size) &&
                    key->prefixlen == prefixlen)
                        return key;
        }
        return NULL;
}

struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
                                         const struct sock *addr_sk)
{
        const union tcp_md5_addr *addr;
        int l3index;

        l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
                                                 addr_sk->sk_bound_dev_if);
        addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
        return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
}
EXPORT_IPV6_MOD(tcp_v4_md5_lookup);

static int tcp_md5sig_info_add(struct sock *sk, gfp_t gfp)
{
        struct tcp_sock *tp = tcp_sk(sk);
        struct tcp_md5sig_info *md5sig;

        md5sig = kmalloc(sizeof(*md5sig), gfp);
        if (!md5sig)
                return -ENOMEM;

        sk_gso_disable(sk);
        INIT_HLIST_HEAD(&md5sig->head);
        rcu_assign_pointer(tp->md5sig_info, md5sig);
        return 0;
}

/* This can be called on a newly created socket, from other files */
static int __tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
                            int family, u8 prefixlen, int l3index, u8 flags,
                            const u8 *newkey, u8 newkeylen, gfp_t gfp)
{
        /* Add Key to the list */
        struct tcp_md5sig_key *key;
        struct tcp_sock *tp = tcp_sk(sk);
        struct tcp_md5sig_info *md5sig;

        key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
        if (key) {
                /* Pre-existing entry - just update that one.
                 * Note that the key might be used concurrently.
                 * data_race() is telling kcsan that we do not care of
                 * key mismatches, since changing MD5 key on live flows
                 * can lead to packet drops.
                 */
                data_race(memcpy(key->key, newkey, newkeylen));

                /* Pairs with READ_ONCE() in tcp_md5_hash_key().
                 * Also note that a reader could catch new key->keylen value
                 * but old key->key[], this is the reason we use __GFP_ZERO
                 * at sock_kmalloc() time below these lines.
                 */
                WRITE_ONCE(key->keylen, newkeylen);

                return 0;
        }

        md5sig = rcu_dereference_protected(tp->md5sig_info,
                                           lockdep_sock_is_held(sk));

        key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
        if (!key)
                return -ENOMEM;

        memcpy(key->key, newkey, newkeylen);
        key->keylen = newkeylen;
        key->family = family;
        key->prefixlen = prefixlen;
        key->l3index = l3index;
        key->flags = flags;
        memcpy(&key->addr, addr,
               (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6) ? sizeof(struct in6_addr) :
                                                                 sizeof(struct in_addr));
        hlist_add_head_rcu(&key->node, &md5sig->head);
        return 0;
}

int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
                   int family, u8 prefixlen, int l3index, u8 flags,
                   const u8 *newkey, u8 newkeylen)
{
        struct tcp_sock *tp = tcp_sk(sk);

        if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
                if (tcp_md5_alloc_sigpool())
                        return -ENOMEM;

                if (tcp_md5sig_info_add(sk, GFP_KERNEL)) {
                        tcp_md5_release_sigpool();
                        return -ENOMEM;
                }

                if (!static_branch_inc(&tcp_md5_needed.key)) {
                        struct tcp_md5sig_info *md5sig;

                        md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
                        rcu_assign_pointer(tp->md5sig_info, NULL);
                        kfree_rcu(md5sig, rcu);
                        tcp_md5_release_sigpool();
                        return -EUSERS;
                }
        }

        return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, flags,
                                newkey, newkeylen, GFP_KERNEL);
}
EXPORT_IPV6_MOD(tcp_md5_do_add);

int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
                     int family, u8 prefixlen, int l3index,
                     struct tcp_md5sig_key *key)
{
        struct tcp_sock *tp = tcp_sk(sk);

        if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) {
                tcp_md5_add_sigpool();

                if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC))) {
                        tcp_md5_release_sigpool();
                        return -ENOMEM;
                }

                if (!static_key_fast_inc_not_disabled(&tcp_md5_needed.key.key)) {
                        struct tcp_md5sig_info *md5sig;

                        md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk));
                        net_warn_ratelimited("Too many TCP-MD5 keys in the system\n");
                        rcu_assign_pointer(tp->md5sig_info, NULL);
                        kfree_rcu(md5sig, rcu);
                        tcp_md5_release_sigpool();
                        return -EUSERS;
                }
        }

        return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index,
                                key->flags, key->key, key->keylen,
                                sk_gfp_mask(sk, GFP_ATOMIC));
}
EXPORT_IPV6_MOD(tcp_md5_key_copy);

int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
                   u8 prefixlen, int l3index, u8 flags)
{
        struct tcp_md5sig_key *key;

        key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
        if (!key)
                return -ENOENT;
        hlist_del_rcu(&key->node);
        atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
        kfree_rcu(key, rcu);
        return 0;
}
EXPORT_IPV6_MOD(tcp_md5_do_del);

void tcp_clear_md5_list(struct sock *sk)
{
        struct tcp_sock *tp = tcp_sk(sk);
        struct tcp_md5sig_key *key;
        struct hlist_node *n;
        struct tcp_md5sig_info *md5sig;

        md5sig = rcu_dereference_protected(tp->md5sig_info, 1);

        hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
                hlist_del(&key->node);
                atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
                kfree(key);
        }
}

static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
                                 sockptr_t optval, int optlen)
{
        struct tcp_md5sig cmd;
        struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
        const union tcp_md5_addr *addr;
        u8 prefixlen = 32;
        int l3index = 0;
        bool l3flag;
        u8 flags;

        if (optlen < sizeof(cmd))
                return -EINVAL;

        if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
                return -EFAULT;

        if (sin->sin_family != AF_INET)
                return -EINVAL;

        flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
        l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;

        if (optname == TCP_MD5SIG_EXT &&
            cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
                prefixlen = cmd.tcpm_prefixlen;
                if (prefixlen > 32)
                        return -EINVAL;
        }

        if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
            cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
                struct net_device *dev;

                rcu_read_lock();
                dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
                if (dev && netif_is_l3_master(dev))
                        l3index = dev->ifindex;

                rcu_read_unlock();

                /* ok to reference set/not set outside of rcu;
                 * right now device MUST be an L3 master
                 */
                if (!dev || !l3index)
                        return -EINVAL;
        }

        addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;

        if (!cmd.tcpm_keylen)
                return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index, flags);

        if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
                return -EINVAL;

        /* Don't allow keys for peers that have a matching TCP-AO key.
         * See the comment in tcp_ao_add_cmd()
         */
        if (tcp_ao_required(sk, addr, AF_INET, l3flag ? l3index : -1, false))
                return -EKEYREJECTED;

        return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags,
                              cmd.tcpm_key, cmd.tcpm_keylen);
}

static int tcp_v4_md5_hash_headers(struct tcp_sigpool *hp,
                                   __be32 daddr, __be32 saddr,
                                   const struct tcphdr *th, int nbytes)
{
        struct tcp4_pseudohdr *bp;
        struct scatterlist sg;
        struct tcphdr *_th;

        bp = hp->scratch;
        bp->saddr = saddr;
        bp->daddr = daddr;
        bp->pad = 0;
        bp->protocol = IPPROTO_TCP;
        bp->len = cpu_to_be16(nbytes);

        _th = (struct tcphdr *)(bp + 1);
        memcpy(_th, th, sizeof(*th));
        _th->check = 0;

        sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
        ahash_request_set_crypt(hp->req, &sg, NULL,
                                sizeof(*bp) + sizeof(*th));
        return crypto_ahash_update(hp->req);
}

static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
                               __be32 daddr, __be32 saddr, const struct tcphdr *th)
{
        struct tcp_sigpool hp;

        if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
                goto clear_hash_nostart;

        if (crypto_ahash_init(hp.req))
                goto clear_hash;
        if (tcp_v4_md5_hash_headers(&hp, daddr, saddr, th, th->doff << 2))
                goto clear_hash;
        if (tcp_md5_hash_key(&hp, key))
                goto clear_hash;
        ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
        if (crypto_ahash_final(hp.req))
                goto clear_hash;

        tcp_sigpool_end(&hp);
        return 0;

clear_hash:
        tcp_sigpool_end(&hp);
clear_hash_nostart:
        memset(md5_hash, 0, 16);
        return 1;
}

int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
                        const struct sock *sk,
                        const struct sk_buff *skb)
{
        const struct tcphdr *th = tcp_hdr(skb);
        struct tcp_sigpool hp;
        __be32 saddr, daddr;

        if (sk) { /* valid for establish/request sockets */
                saddr = sk->sk_rcv_saddr;
                daddr = sk->sk_daddr;
        } else {
                const struct iphdr *iph = ip_hdr(skb);
                saddr = iph->saddr;
                daddr = iph->daddr;
        }

        if (tcp_sigpool_start(tcp_md5_sigpool_id, &hp))
                goto clear_hash_nostart;

        if (crypto_ahash_init(hp.req))
                goto clear_hash;

        if (tcp_v4_md5_hash_headers(&hp, daddr, saddr, th, skb->len))
                goto clear_hash;
        if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
                goto clear_hash;
        if (tcp_md5_hash_key(&hp, key))
                goto clear_hash;
        ahash_request_set_crypt(hp.req, NULL, md5_hash, 0);
        if (crypto_ahash_final(hp.req))
                goto clear_hash;

        tcp_sigpool_end(&hp);
        return 0;

clear_hash:
        tcp_sigpool_end(&hp);
clear_hash_nostart:
        memset(md5_hash, 0, 16);
        return 1;
}
EXPORT_IPV6_MOD(tcp_v4_md5_hash_skb);

#endif

static void tcp_v4_init_req(struct request_sock *req,
                            const struct sock *sk_listener,
                            struct sk_buff *skb)
{
        struct inet_request_sock *ireq = inet_rsk(req);
        struct net *net = sock_net(sk_listener);

        sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
        sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
        RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
}

static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
                                          struct sk_buff *skb,
                                          struct flowi *fl,
                                          struct request_sock *req,
                                          u32 tw_isn)
{
        tcp_v4_init_req(req, sk, skb);

        if (security_inet_conn_request(sk, skb, req))
                return NULL;

        return inet_csk_route_req(sk, &fl->u.ip4, req);
}

struct request_sock_ops tcp_request_sock_ops __read_mostly = {
        .family                =        PF_INET,
        .obj_size        =        sizeof(struct tcp_request_sock),
        .send_ack        =        tcp_v4_reqsk_send_ack,
        .destructor        =        tcp_v4_reqsk_destructor,
        .send_reset        =        tcp_v4_send_reset,
        .syn_ack_timeout =        tcp_syn_ack_timeout,
};

const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
        .mss_clamp        =        TCP_MSS_DEFAULT,
#ifdef CONFIG_TCP_MD5SIG
        .req_md5_lookup        =        tcp_v4_md5_lookup,
        .calc_md5_hash        =        tcp_v4_md5_hash_skb,
#endif
#ifdef CONFIG_TCP_AO
        .ao_lookup        =        tcp_v4_ao_lookup_rsk,
        .ao_calc_key        =        tcp_v4_ao_calc_key_rsk,
        .ao_synack_hash        =        tcp_v4_ao_synack_hash,
#endif
#ifdef CONFIG_SYN_COOKIES
        .cookie_init_seq =        cookie_v4_init_sequence,
#endif
        .route_req        =        tcp_v4_route_req,
        .init_seq        =        tcp_v4_init_seq,
        .init_ts_off        =        tcp_v4_init_ts_off,
        .send_synack        =        tcp_v4_send_synack,
};

int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
{
        /* Never answer to SYNs send to broadcast or multicast */
        if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
                goto drop;

        return tcp_conn_request(&tcp_request_sock_ops,
                                &tcp_request_sock_ipv4_ops, sk, skb);

drop:
        tcp_listendrop(sk);
        return 0;
}
EXPORT_IPV6_MOD(tcp_v4_conn_request);


/*
 * The three way handshake has completed - we got a valid synack -
 * now create the new socket.
 */
struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
                                  struct request_sock *req,
                                  struct dst_entry *dst,
                                  struct request_sock *req_unhash,
                                  bool *own_req)
{
        struct inet_request_sock *ireq;
        bool found_dup_sk = false;
        struct inet_sock *newinet;
        struct tcp_sock *newtp;
        struct sock *newsk;
#ifdef CONFIG_TCP_MD5SIG
        const union tcp_md5_addr *addr;
        struct tcp_md5sig_key *key;
        int l3index;
#endif
        struct ip_options_rcu *inet_opt;

        if (sk_acceptq_is_full(sk))
                goto exit_overflow;

        newsk = tcp_create_openreq_child(sk, req, skb);
        if (!newsk)
                goto exit_nonewsk;

        newsk->sk_gso_type = SKB_GSO_TCPV4;
        inet_sk_rx_dst_set(newsk, skb);

        newtp                      = tcp_sk(newsk);
        newinet                      = inet_sk(newsk);
        ireq                      = inet_rsk(req);
        inet_opt              = rcu_dereference(ireq->ireq_opt);
        RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
        newinet->mc_index     = inet_iif(skb);
        newinet->mc_ttl              = ip_hdr(skb)->ttl;
        newinet->rcv_tos      = ip_hdr(skb)->tos;
        inet_csk(newsk)->icsk_ext_hdr_len = 0;
        if (inet_opt)
                inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
        atomic_set(&newinet->inet_id, get_random_u16());

        /* Set ToS of the new socket based upon the value of incoming SYN.
         * ECT bits are set later in tcp_init_transfer().
         */
        if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
                newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;

        if (!dst) {
                dst = inet_csk_route_child_sock(sk, newsk, req);
                if (!dst)
                        goto put_and_exit;
        } else {
                /* syncookie case : see end of cookie_v4_check() */
        }
        sk_setup_caps(newsk, dst);

        tcp_ca_openreq_child(newsk, dst);

        tcp_sync_mss(newsk, dst_mtu(dst));
        newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));

        tcp_initialize_rcv_mss(newsk);

#ifdef CONFIG_TCP_MD5SIG
        l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
        /* Copy over the MD5 key from the original socket */
        addr = (union tcp_md5_addr *)&newinet->inet_daddr;
        key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
        if (key && !tcp_rsk_used_ao(req)) {
                if (tcp_md5_key_copy(newsk, addr, AF_INET, 32, l3index, key))
                        goto put_and_exit;
                sk_gso_disable(newsk);
        }
#endif
#ifdef CONFIG_TCP_AO
        if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET))
                goto put_and_exit; /* OOM, release back memory */
#endif

        if (__inet_inherit_port(sk, newsk) < 0)
                goto put_and_exit;
        *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
                                       &found_dup_sk);
        if (likely(*own_req)) {
                tcp_move_syn(newtp, req);
                ireq->ireq_opt = NULL;
        } else {
                newinet->inet_opt = NULL;

                if (!req_unhash && found_dup_sk) {
                        /* This code path should only be executed in the
                         * syncookie case only
                         */
                        bh_unlock_sock(newsk);
                        sock_put(newsk);
                        newsk = NULL;
                }
        }
        return newsk;

exit_overflow:
        NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
exit_nonewsk:
        dst_release(dst);
exit:
        tcp_listendrop(sk);
        return NULL;
put_and_exit:
        newinet->inet_opt = NULL;
        inet_csk_prepare_forced_close(newsk);
        tcp_done(newsk);
        goto exit;
}
EXPORT_IPV6_MOD(tcp_v4_syn_recv_sock);

static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
{
#ifdef CONFIG_SYN_COOKIES
        const struct tcphdr *th = tcp_hdr(skb);

        if (!th->syn)
                sk = cookie_v4_check(sk, skb);
#endif
        return sk;
}

u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
                         struct tcphdr *th, u32 *cookie)
{
        u16 mss = 0;
#ifdef CONFIG_SYN_COOKIES
        mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
                                    &tcp_request_sock_ipv4_ops, sk, th);
        if (mss) {
                *cookie = __cookie_v4_init_sequence(iph, th, &mss);
                tcp_synq_overflow(sk);
        }
#endif
        return mss;
}

INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
                                                           u32));
/* The socket must have it's spinlock held when we get
 * here, unless it is a TCP_LISTEN socket.
 *
 * We have a potential double-lock case here, so even when
 * doing backlog processing we use the BH locking scheme.
 * This is because we cannot sleep with the original spinlock
 * held.
 */
int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
{
        enum skb_drop_reason reason;
        struct sock *rsk;

        reason = psp_sk_rx_policy_check(sk, skb);
        if (reason)
                goto err_discard;

        if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
                struct dst_entry *dst;

                dst = rcu_dereference_protected(sk->sk_rx_dst,
                                                lockdep_sock_is_held(sk));

                sock_rps_save_rxhash(sk, skb);
                sk_mark_napi_id(sk, skb);
                if (dst) {
                        if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
                            !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
                                             dst, 0)) {
                                RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
                                dst_release(dst);
                        }
                }
                tcp_rcv_established(sk, skb);
                return 0;
        }

        if (tcp_checksum_complete(skb))
                goto csum_err;

        if (sk->sk_state == TCP_LISTEN) {
                struct sock *nsk = tcp_v4_cookie_check(sk, skb);

                if (!nsk)
                        return 0;
                if (nsk != sk) {
                        reason = tcp_child_process(sk, nsk, skb);
                        if (reason) {
                                rsk = nsk;
                                goto reset;
                        }
                        return 0;
                }
        } else
                sock_rps_save_rxhash(sk, skb);

        reason = tcp_rcv_state_process(sk, skb);
        if (reason) {
                rsk = sk;
                goto reset;
        }
        return 0;

reset:
        tcp_v4_send_reset(rsk, skb, sk_rst_convert_drop_reason(reason));
discard:
        sk_skb_reason_drop(sk, skb, reason);
        /* Be careful here. If this function gets more complicated and
         * gcc suffers from register pressure on the x86, sk (in %ebx)
         * might be destroyed here. This current version compiles correctly,
         * but you have been warned.
         */
        return 0;

csum_err:
        reason = SKB_DROP_REASON_TCP_CSUM;
        trace_tcp_bad_csum(skb);
        TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
err_discard:
        TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
        goto discard;
}
EXPORT_SYMBOL(tcp_v4_do_rcv);

int tcp_v4_early_demux(struct sk_buff *skb)
{
        struct net *net = dev_net_rcu(skb->dev);
        const struct iphdr *iph;
        const struct tcphdr *th;
        struct sock *sk;

        if (skb->pkt_type != PACKET_HOST)
                return 0;

        if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
                return 0;

        iph = ip_hdr(skb);
        th = tcp_hdr(skb);

        if (th->doff < sizeof(struct tcphdr) / 4)
                return 0;

        sk = __inet_lookup_established(net, iph->saddr, th->source,
                                       iph->daddr, ntohs(th->dest),
                                       skb->skb_iif, inet_sdif(skb));
        if (sk) {
                skb->sk = sk;
                skb->destructor = sock_edemux;
                if (sk_fullsock(sk)) {
                        struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);

                        if (dst)
                                dst = dst_check(dst, 0);
                        if (dst &&
                            sk->sk_rx_dst_ifindex == skb->skb_iif)
                                skb_dst_set_noref(skb, dst);
                }
        }
        return 0;
}

bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
                     enum skb_drop_reason *reason)
{
        u32 tail_gso_size, tail_gso_segs;
        struct skb_shared_info *shinfo;
        const struct tcphdr *th;
        struct tcphdr *thtail;
        struct sk_buff *tail;
        unsigned int hdrlen;
        bool fragstolen;
        u32 gso_segs;
        u32 gso_size;
        u64 limit;
        int delta;
        int err;

        /* In case all data was pulled from skb frags (in __pskb_pull_tail()),
         * we can fix skb->truesize to its real value to avoid future drops.
         * This is valid because skb is not yet charged to the socket.
         * It has been noticed pure SACK packets were sometimes dropped
         * (if cooked by drivers without copybreak feature).
         */
        skb_condense(skb);

        tcp_cleanup_skb(skb);

        if (unlikely(tcp_checksum_complete(skb))) {
                bh_unlock_sock(sk);
                trace_tcp_bad_csum(skb);
                *reason = SKB_DROP_REASON_TCP_CSUM;
                __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
                __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
                return true;
        }

        /* Attempt coalescing to last skb in backlog, even if we are
         * above the limits.
         * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
         */
        th = (const struct tcphdr *)skb->data;
        hdrlen = th->doff * 4;

        tail = sk->sk_backlog.tail;
        if (!tail)
                goto no_coalesce;
        thtail = (struct tcphdr *)tail->data;

        if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
            TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
            ((TCP_SKB_CB(tail)->tcp_flags |
              TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
            !((TCP_SKB_CB(tail)->tcp_flags &
              TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
            ((TCP_SKB_CB(tail)->tcp_flags ^
              TCP_SKB_CB(skb)->tcp_flags) &
             (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)) ||
            !tcp_skb_can_collapse_rx(tail, skb) ||
            thtail->doff != th->doff ||
            memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)) ||
            /* prior to PSP Rx policy check, retain exact PSP metadata */
            psp_skb_coalesce_diff(tail, skb))
                goto no_coalesce;

        __skb_pull(skb, hdrlen);

        shinfo = skb_shinfo(skb);
        gso_size = shinfo->gso_size ?: skb->len;
        gso_segs = shinfo->gso_segs ?: 1;

        shinfo = skb_shinfo(tail);
        tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
        tail_gso_segs = shinfo->gso_segs ?: 1;

        if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
                TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;

                if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
                        TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
                        thtail->window = th->window;
                }

                /* We have to update both TCP_SKB_CB(tail)->tcp_flags and
                 * thtail->fin, so that the fast path in tcp_rcv_established()
                 * is not entered if we append a packet with a FIN.
                 * SYN, RST, URG are not present.
                 * ACK is set on both packets.
                 * PSH : we do not really care in TCP stack,
                 *       at least for 'GRO' packets.
                 */
                thtail->fin |= th->fin;
                TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;

                if (TCP_SKB_CB(skb)->has_rxtstamp) {
                        TCP_SKB_CB(tail)->has_rxtstamp = true;
                        tail->tstamp = skb->tstamp;
                        skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
                }

                /* Not as strict as GRO. We only need to carry mss max value */
                shinfo->gso_size = max(gso_size, tail_gso_size);
                shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);

                sk->sk_backlog.len += delta;
                __NET_INC_STATS(sock_net(sk),
                                LINUX_MIB_TCPBACKLOGCOALESCE);
                kfree_skb_partial(skb, fragstolen);
                return false;
        }
        __skb_push(skb, hdrlen);

no_coalesce:
        /* sk->sk_backlog.len is reset only at the end of __release_sock().
         * Both sk->sk_backlog.len and sk->sk_rmem_alloc could reach
         * sk_rcvbuf in normal conditions.
         */
        limit = ((u64)READ_ONCE(sk->sk_rcvbuf)) << 1;

        limit += ((u32)READ_ONCE(sk->sk_sndbuf)) >> 1;

        /* Only socket owner can try to collapse/prune rx queues
         * to reduce memory overhead, so add a little headroom here.
         * Few sockets backlog are possibly concurrently non empty.
         */
        limit += 64 * 1024;

        limit = min_t(u64, limit, UINT_MAX);

        err = sk_add_backlog(sk, skb, limit);
        if (unlikely(err)) {
                bh_unlock_sock(sk);
                if (err == -ENOMEM) {
                        *reason = SKB_DROP_REASON_PFMEMALLOC;
                        __NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
                } else {
                        *reason = SKB_DROP_REASON_SOCKET_BACKLOG;
                        __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
                }
                return true;
        }
        return false;
}
EXPORT_IPV6_MOD(tcp_add_backlog);

int tcp_filter(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason *reason)
{
        struct tcphdr *th = (struct tcphdr *)skb->data;

        return sk_filter_trim_cap(sk, skb, th->doff * 4, reason);
}
EXPORT_IPV6_MOD(tcp_filter);

static void tcp_v4_restore_cb(struct sk_buff *skb)
{
        memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
                sizeof(struct inet_skb_parm));
}

static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
                           const struct tcphdr *th)
{
        /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
         * barrier() makes sure compiler wont play fool^Waliasing games.
         */
        memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
                sizeof(struct inet_skb_parm));
        barrier();

        TCP_SKB_CB(skb)->seq = ntohl(th->seq);
        TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
                                    skb->len - th->doff * 4);
        TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
        TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th);
        TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
        TCP_SKB_CB(skb)->sacked         = 0;
        TCP_SKB_CB(skb)->has_rxtstamp =
                        skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
}

/*
 *        From tcp_input.c
 */

int tcp_v4_rcv(struct sk_buff *skb)
{
        struct net *net = dev_net_rcu(skb->dev);
        enum skb_drop_reason drop_reason;
        enum tcp_tw_status tw_status;
        int sdif = inet_sdif(skb);
        int dif = inet_iif(skb);
        const struct iphdr *iph;
        const struct tcphdr *th;
        struct sock *sk = NULL;
        bool refcounted;
        int ret;
        u32 isn;

        drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
        if (skb->pkt_type != PACKET_HOST)
                goto discard_it;

        /* Count it even if it's bad */
        __TCP_INC_STATS(net, TCP_MIB_INSEGS);

        if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
                goto discard_it;

        th = (const struct tcphdr *)skb->data;

        if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
                drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
                goto bad_packet;
        }
        if (!pskb_may_pull(skb, th->doff * 4))
                goto discard_it;

        /* An explanation is required here, I think.
         * Packet length and doff are validated by header prediction,
         * provided case of th->doff==0 is eliminated.
         * So, we defer the checks. */

        if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
                goto csum_error;

        th = (const struct tcphdr *)skb->data;
        iph = ip_hdr(skb);
lookup:
        sk = __inet_lookup_skb(skb, __tcp_hdrlen(th), th->source,
                               th->dest, sdif, &refcounted);
        if (!sk)
                goto no_tcp_socket;

        if (sk->sk_state == TCP_TIME_WAIT)
                goto do_time_wait;

        if (sk->sk_state == TCP_NEW_SYN_RECV) {
                struct request_sock *req = inet_reqsk(sk);
                bool req_stolen = false;
                struct sock *nsk;

                sk = req->rsk_listener;
                if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
                        drop_reason = SKB_DROP_REASON_XFRM_POLICY;
                else
                        drop_reason = tcp_inbound_hash(sk, req, skb,
                                                       &iph->saddr, &iph->daddr,
                                                       AF_INET, dif, sdif);
                if (unlikely(drop_reason)) {
                        sk_drops_skbadd(sk, skb);
                        reqsk_put(req);
                        goto discard_it;
                }
                if (tcp_checksum_complete(skb)) {
                        reqsk_put(req);
                        goto csum_error;
                }
                if (unlikely(sk->sk_state != TCP_LISTEN)) {
                        nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
                        if (!nsk) {
                                inet_csk_reqsk_queue_drop_and_put(sk, req);
                                goto lookup;
                        }
                        sk = nsk;
                        /* reuseport_migrate_sock() has already held one sk_refcnt
                         * before returning.
                         */
                } else {
                        /* We own a reference on the listener, increase it again
                         * as we might lose it too soon.
                         */
                        sock_hold(sk);
                }
                refcounted = true;
                nsk = NULL;
                if (!tcp_filter(sk, skb, &drop_reason)) {
                        th = (const struct tcphdr *)skb->data;
                        iph = ip_hdr(skb);
                        tcp_v4_fill_cb(skb, iph, th);
                        nsk = tcp_check_req(sk, skb, req, false, &req_stolen,
                                            &drop_reason);
                }
                if (!nsk) {
                        reqsk_put(req);
                        if (req_stolen) {
                                /* Another cpu got exclusive access to req
                                 * and created a full blown socket.
                                 * Try to feed this packet to this socket
                                 * instead of discarding it.
                                 */
                                tcp_v4_restore_cb(skb);
                                sock_put(sk);
                                goto lookup;
                        }
                        goto discard_and_relse;
                }
                nf_reset_ct(skb);
                if (nsk == sk) {
                        reqsk_put(req);
                        tcp_v4_restore_cb(skb);
                } else {
                        drop_reason = tcp_child_process(sk, nsk, skb);
                        if (drop_reason) {
                                enum sk_rst_reason rst_reason;

                                rst_reason = sk_rst_convert_drop_reason(drop_reason);
                                tcp_v4_send_reset(nsk, skb, rst_reason);
                                goto discard_and_relse;
                        }
                        sock_put(sk);
                        return 0;
                }
        }

process:
        if (static_branch_unlikely(&ip4_min_ttl)) {
                /* min_ttl can be changed concurrently from do_ip_setsockopt() */
                if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
                        __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
                        drop_reason = SKB_DROP_REASON_TCP_MINTTL;
                        goto discard_and_relse;
                }
        }

        if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
                drop_reason = SKB_DROP_REASON_XFRM_POLICY;
                goto discard_and_relse;
        }

        drop_reason = tcp_inbound_hash(sk, NULL, skb, &iph->saddr, &iph->daddr,
                                       AF_INET, dif, sdif);
        if (drop_reason)
                goto discard_and_relse;

        nf_reset_ct(skb);

        if (tcp_filter(sk, skb, &drop_reason))
                goto discard_and_relse;

        th = (const struct tcphdr *)skb->data;
        iph = ip_hdr(skb);
        tcp_v4_fill_cb(skb, iph, th);

        skb->dev = NULL;

        if (sk->sk_state == TCP_LISTEN) {
                ret = tcp_v4_do_rcv(sk, skb);
                goto put_and_return;
        }

        sk_incoming_cpu_update(sk);

        bh_lock_sock_nested(sk);
        tcp_segs_in(tcp_sk(sk), skb);
        ret = 0;
        if (!sock_owned_by_user(sk)) {
                ret = tcp_v4_do_rcv(sk, skb);
        } else {
                if (tcp_add_backlog(sk, skb, &drop_reason))
                        goto discard_and_relse;
        }
        bh_unlock_sock(sk);

put_and_return:
        if (refcounted)
                sock_put(sk);

        return ret;

no_tcp_socket:
        drop_reason = SKB_DROP_REASON_NO_SOCKET;
        if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
                goto discard_it;

        tcp_v4_fill_cb(skb, iph, th);

        if (tcp_checksum_complete(skb)) {
csum_error:
                drop_reason = SKB_DROP_REASON_TCP_CSUM;
                trace_tcp_bad_csum(skb);
                __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
bad_packet:
                __TCP_INC_STATS(net, TCP_MIB_INERRS);
        } else {
                tcp_v4_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason));
        }

discard_it:
        SKB_DR_OR(drop_reason, NOT_SPECIFIED);
        /* Discard frame. */
        sk_skb_reason_drop(sk, skb, drop_reason);
        return 0;

discard_and_relse:
        sk_drops_skbadd(sk, skb);
        if (refcounted)
                sock_put(sk);
        goto discard_it;

do_time_wait:
        if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
                drop_reason = SKB_DROP_REASON_XFRM_POLICY;
                inet_twsk_put(inet_twsk(sk));
                goto discard_it;
        }

        tcp_v4_fill_cb(skb, iph, th);

        if (tcp_checksum_complete(skb)) {
                inet_twsk_put(inet_twsk(sk));
                goto csum_error;
        }

        tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn,
                                               &drop_reason);
        switch (tw_status) {
        case TCP_TW_SYN: {
                struct sock *sk2 = inet_lookup_listener(net, skb, __tcp_hdrlen(th),
                                                        iph->saddr, th->source,
                                                        iph->daddr, th->dest,
                                                        inet_iif(skb),
                                                        sdif);
                if (sk2) {
                        inet_twsk_deschedule_put(inet_twsk(sk));
                        sk = sk2;
                        tcp_v4_restore_cb(skb);
                        refcounted = false;
                        __this_cpu_write(tcp_tw_isn, isn);
                        goto process;
                }

                drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb);
                if (drop_reason)
                        break;
        }
                /* to ACK */
                fallthrough;
        case TCP_TW_ACK:
        case TCP_TW_ACK_OOW:
                tcp_v4_timewait_ack(sk, skb, tw_status);
                break;
        case TCP_TW_RST:
                tcp_v4_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET);
                inet_twsk_deschedule_put(inet_twsk(sk));
                goto discard_it;
        case TCP_TW_SUCCESS:;
        }
        goto discard_it;
}

static struct timewait_sock_ops tcp_timewait_sock_ops = {
        .twsk_obj_size        = sizeof(struct tcp_timewait_sock),
};

void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
{
        struct dst_entry *dst = skb_dst(skb);

        if (dst && dst_hold_safe(dst)) {
                rcu_assign_pointer(sk->sk_rx_dst, dst);
                sk->sk_rx_dst_ifindex = skb->skb_iif;
        }
}
EXPORT_IPV6_MOD(inet_sk_rx_dst_set);

const struct inet_connection_sock_af_ops ipv4_specific = {
        .queue_xmit           = ip_queue_xmit,
        .send_check           = tcp_v4_send_check,
        .rebuild_header           = inet_sk_rebuild_header,
        .sk_rx_dst_set           = inet_sk_rx_dst_set,
        .conn_request           = tcp_v4_conn_request,
        .syn_recv_sock           = tcp_v4_syn_recv_sock,
        .net_header_len           = sizeof(struct iphdr),
        .setsockopt           = ip_setsockopt,
        .getsockopt           = ip_getsockopt,
        .mtu_reduced           = tcp_v4_mtu_reduced,
};
EXPORT_IPV6_MOD(ipv4_specific);

#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
#ifdef CONFIG_TCP_MD5SIG
        .md5_lookup                = tcp_v4_md5_lookup,
        .calc_md5_hash                = tcp_v4_md5_hash_skb,
        .md5_parse                = tcp_v4_parse_md5_keys,
#endif
#ifdef CONFIG_TCP_AO
        .ao_lookup                = tcp_v4_ao_lookup,
        .calc_ao_hash                = tcp_v4_ao_hash_skb,
        .ao_parse                = tcp_v4_parse_ao,
        .ao_calc_key_sk                = tcp_v4_ao_calc_key_sk,
#endif
};

static void tcp4_destruct_sock(struct sock *sk)
{
        tcp_md5_destruct_sock(sk);
        tcp_ao_destroy_sock(sk, false);
        inet_sock_destruct(sk);
}
#endif

/* NOTE: A lot of things set to zero explicitly by call to
 *       sk_alloc() so need not be done here.
 */
static int tcp_v4_init_sock(struct sock *sk)
{
        struct inet_connection_sock *icsk = inet_csk(sk);

        tcp_init_sock(sk);

        icsk->icsk_af_ops = &ipv4_specific;

#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
        tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
        sk->sk_destruct = tcp4_destruct_sock;
#endif

        return 0;
}

static void tcp_release_user_frags(struct sock *sk)
{
#ifdef CONFIG_PAGE_POOL
        unsigned long index;
        void *netmem;

        xa_for_each(&sk->sk_user_frags, index, netmem)
                WARN_ON_ONCE(!napi_pp_put_page((__force netmem_ref)netmem));
#endif
}

void tcp_v4_destroy_sock(struct sock *sk)
{
        struct tcp_sock *tp = tcp_sk(sk);

        tcp_release_user_frags(sk);

        xa_destroy(&sk->sk_user_frags);

        trace_tcp_destroy_sock(sk);

        tcp_clear_xmit_timers(sk);

        tcp_cleanup_congestion_control(sk);

        tcp_cleanup_ulp(sk);

        /* Cleanup up the write buffer. */
        tcp_write_queue_purge(sk);

        /* Check if we want to disable active TFO */
        tcp_fastopen_active_disable_ofo_check(sk);

        /* Cleans up our, hopefully empty, out_of_order_queue. */
        skb_rbtree_purge(&tp->out_of_order_queue);

        /* Clean up a referenced TCP bind bucket. */
        if (inet_csk(sk)->icsk_bind_hash)
                inet_put_port(sk);

        BUG_ON(rcu_access_pointer(tp->fastopen_rsk));

        /* If socket is aborted during connect operation */
        tcp_free_fastopen_req(tp);
        tcp_fastopen_destroy_cipher(sk);
        tcp_saved_syn_free(tp);

        sk_sockets_allocated_dec(sk);
}
EXPORT_IPV6_MOD(tcp_v4_destroy_sock);

#ifdef CONFIG_PROC_FS
/* Proc filesystem TCP sock list dumping. */

static unsigned short seq_file_family(const struct seq_file *seq);

static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
{
        unsigned short family = seq_file_family(seq);

        /* AF_UNSPEC is used as a match all */
        return ((family == AF_UNSPEC || family == sk->sk_family) &&
                net_eq(sock_net(sk), seq_file_net(seq)));
}

/* Find a non empty bucket (starting from st->bucket)
 * and return the first sk from it.
 */
static void *listening_get_first(struct seq_file *seq)
{
        struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
        struct tcp_iter_state *st = seq->private;

        st->offset = 0;
        for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
                struct inet_listen_hashbucket *ilb2;
                struct hlist_nulls_node *node;
                struct sock *sk;

                ilb2 = &hinfo->lhash2[st->bucket];
                if (hlist_nulls_empty(&ilb2->nulls_head))
                        continue;

                spin_lock(&ilb2->lock);
                sk_nulls_for_each(sk, node, &ilb2->nulls_head) {
                        if (seq_sk_match(seq, sk))
                                return sk;
                }
                spin_unlock(&ilb2->lock);
        }

        return NULL;
}

/* Find the next sk of "cur" within the same bucket (i.e. st->bucket).
 * If "cur" is the last one in the st->bucket,
 * call listening_get_first() to return the first sk of the next
 * non empty bucket.
 */
static void *listening_get_next(struct seq_file *seq, void *cur)
{
        struct tcp_iter_state *st = seq->private;
        struct inet_listen_hashbucket *ilb2;
        struct hlist_nulls_node *node;
        struct inet_hashinfo *hinfo;
        struct sock *sk = cur;

        ++st->num;
        ++st->offset;

        sk = sk_nulls_next(sk);
        sk_nulls_for_each_from(sk, node) {
                if (seq_sk_match(seq, sk))
                        return sk;
        }

        hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
        ilb2 = &hinfo->lhash2[st->bucket];
        spin_unlock(&ilb2->lock);
        ++st->bucket;
        return listening_get_first(seq);
}

static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
{
        struct tcp_iter_state *st = seq->private;
        void *rc;

        st->bucket = 0;
        st->offset = 0;
        rc = listening_get_first(seq);

        while (rc && *pos) {
                rc = listening_get_next(seq, rc);
                --*pos;
        }
        return rc;
}

static inline bool empty_bucket(struct inet_hashinfo *hinfo,
                                const struct tcp_iter_state *st)
{
        return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
}

/*
 * Get first established socket starting from bucket given in st->bucket.
 * If st->bucket is zero, the very first socket in the hash is returned.
 */
static void *established_get_first(struct seq_file *seq)
{
        struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
        struct tcp_iter_state *st = seq->private;

        st->offset = 0;
        for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
                struct sock *sk;
                struct hlist_nulls_node *node;
                spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);

                cond_resched();

                /* Lockless fast path for the common case of empty buckets */
                if (empty_bucket(hinfo, st))
                        continue;

                spin_lock_bh(lock);
                sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
                        if (seq_sk_match(seq, sk))
                                return sk;
                }
                spin_unlock_bh(lock);
        }

        return NULL;
}

static void *established_get_next(struct seq_file *seq, void *cur)
{
        struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
        struct tcp_iter_state *st = seq->private;
        struct hlist_nulls_node *node;
        struct sock *sk = cur;

        ++st->num;
        ++st->offset;

        sk = sk_nulls_next(sk);

        sk_nulls_for_each_from(sk, node) {
                if (seq_sk_match(seq, sk))
                        return sk;
        }

        spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
        ++st->bucket;
        return established_get_first(seq);
}

static void *established_get_idx(struct seq_file *seq, loff_t pos)
{
        struct tcp_iter_state *st = seq->private;
        void *rc;

        st->bucket = 0;
        rc = established_get_first(seq);

        while (rc && pos) {
                rc = established_get_next(seq, rc);
                --pos;
        }
        return rc;
}

static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
{
        void *rc;
        struct tcp_iter_state *st = seq->private;

        st->state = TCP_SEQ_STATE_LISTENING;
        rc          = listening_get_idx(seq, &pos);

        if (!rc) {
                st->state = TCP_SEQ_STATE_ESTABLISHED;
                rc          = established_get_idx(seq, pos);
        }

        return rc;
}

static void *tcp_seek_last_pos(struct seq_file *seq)
{
        struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
        struct tcp_iter_state *st = seq->private;
        int bucket = st->bucket;
        int offset = st->offset;
        int orig_num = st->num;
        void *rc = NULL;

        switch (st->state) {
        case TCP_SEQ_STATE_LISTENING:
                if (st->bucket > hinfo->lhash2_mask)
                        break;
                rc = listening_get_first(seq);
                while (offset-- && rc && bucket == st->bucket)
                        rc = listening_get_next(seq, rc);
                if (rc)
                        break;
                st->bucket = 0;
                st->state = TCP_SEQ_STATE_ESTABLISHED;
                fallthrough;
        case TCP_SEQ_STATE_ESTABLISHED:
                if (st->bucket > hinfo->ehash_mask)
                        break;
                rc = established_get_first(seq);
                while (offset-- && rc && bucket == st->bucket)
                        rc = established_get_next(seq, rc);
        }

        st->num = orig_num;

        return rc;
}

void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
{
        struct tcp_iter_state *st = seq->private;
        void *rc;

        if (*pos && *pos == st->last_pos) {
                rc = tcp_seek_last_pos(seq);
                if (rc)
                        goto out;
        }

        st->state = TCP_SEQ_STATE_LISTENING;
        st->num = 0;
        st->bucket = 0;
        st->offset = 0;
        rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;

out:
        st->last_pos = *pos;
        return rc;
}
EXPORT_IPV6_MOD(tcp_seq_start);

void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct tcp_iter_state *st = seq->private;
        void *rc = NULL;

        if (v == SEQ_START_TOKEN) {
                rc = tcp_get_idx(seq, 0);
                goto out;
        }

        switch (st->state) {
        case TCP_SEQ_STATE_LISTENING:
                rc = listening_get_next(seq, v);
                if (!rc) {
                        st->state = TCP_SEQ_STATE_ESTABLISHED;
                        st->bucket = 0;
                        st->offset = 0;
                        rc          = established_get_first(seq);
                }
                break;
        case TCP_SEQ_STATE_ESTABLISHED:
                rc = established_get_next(seq, v);
                break;
        }
out:
        ++*pos;
        st->last_pos = *pos;
        return rc;
}
EXPORT_IPV6_MOD(tcp_seq_next);

void tcp_seq_stop(struct seq_file *seq, void *v)
{
        struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
        struct tcp_iter_state *st = seq->private;

        switch (st->state) {
        case TCP_SEQ_STATE_LISTENING:
                if (v != SEQ_START_TOKEN)
                        spin_unlock(&hinfo->lhash2[st->bucket].lock);
                break;
        case TCP_SEQ_STATE_ESTABLISHED:
                if (v)
                        spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
                break;
        }
}
EXPORT_IPV6_MOD(tcp_seq_stop);

static void get_openreq4(const struct request_sock *req,
                         struct seq_file *f, int i)
{
        const struct inet_request_sock *ireq = inet_rsk(req);
        long delta = req->rsk_timer.expires - jiffies;

        seq_printf(f, "%4d: %08X:%04X %08X:%04X"
                " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
                i,
                ireq->ir_loc_addr,
                ireq->ir_num,
                ireq->ir_rmt_addr,
                ntohs(ireq->ir_rmt_port),
                TCP_SYN_RECV,
                0, 0, /* could print option size, but that is af dependent. */
                1,    /* timers active (only the expire timer) */
                jiffies_delta_to_clock_t(delta),
                req->num_timeout,
                from_kuid_munged(seq_user_ns(f),
                                 sk_uid(req->rsk_listener)),
                0,  /* non standard timer */
                0, /* open_requests have no inode */
                0,
                req);
}

static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
{
        int timer_active;
        unsigned long timer_expires;
        const struct tcp_sock *tp = tcp_sk(sk);
        const struct inet_connection_sock *icsk = inet_csk(sk);
        const struct inet_sock *inet = inet_sk(sk);
        const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
        __be32 dest = inet->inet_daddr;
        __be32 src = inet->inet_rcv_saddr;
        __u16 destp = ntohs(inet->inet_dport);
        __u16 srcp = ntohs(inet->inet_sport);
        u8 icsk_pending;
        int rx_queue;
        int state;

        icsk_pending = smp_load_acquire(&icsk->icsk_pending);
        if (icsk_pending == ICSK_TIME_RETRANS ||
            icsk_pending == ICSK_TIME_REO_TIMEOUT ||
            icsk_pending == ICSK_TIME_LOSS_PROBE) {
                timer_active        = 1;
                timer_expires        = icsk_timeout(icsk);
        } else if (icsk_pending == ICSK_TIME_PROBE0) {
                timer_active        = 4;
                timer_expires        = icsk_timeout(icsk);
        } else if (timer_pending(&sk->sk_timer)) {
                timer_active        = 2;
                timer_expires        = sk->sk_timer.expires;
        } else {
                timer_active        = 0;
                timer_expires = jiffies;
        }

        state = inet_sk_state_load(sk);
        if (state == TCP_LISTEN)
                rx_queue = READ_ONCE(sk->sk_ack_backlog);
        else
                /* Because we don't lock the socket,
                 * we might find a transient negative value.
                 */
                rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
                                      READ_ONCE(tp->copied_seq), 0);

        seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
                        "%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
                i, src, srcp, dest, destp, state,
                READ_ONCE(tp->write_seq) - tp->snd_una,
                rx_queue,
                timer_active,
                jiffies_delta_to_clock_t(timer_expires - jiffies),
                READ_ONCE(icsk->icsk_retransmits),
                from_kuid_munged(seq_user_ns(f), sk_uid(sk)),
                READ_ONCE(icsk->icsk_probes_out),
                sock_i_ino(sk),
                refcount_read(&sk->sk_refcnt), sk,
                jiffies_to_clock_t(icsk->icsk_rto),
                jiffies_to_clock_t(icsk->icsk_ack.ato),
                (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
                tcp_snd_cwnd(tp),
                state == TCP_LISTEN ?
                    fastopenq->max_qlen :
                    (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
}

static void get_timewait4_sock(const struct inet_timewait_sock *tw,
                               struct seq_file *f, int i)
{
        long delta = tw->tw_timer.expires - jiffies;
        __be32 dest, src;
        __u16 destp, srcp;

        dest  = tw->tw_daddr;
        src   = tw->tw_rcv_saddr;
        destp = ntohs(tw->tw_dport);
        srcp  = ntohs(tw->tw_sport);

        seq_printf(f, "%4d: %08X:%04X %08X:%04X"
                " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
                i, src, srcp, dest, destp, READ_ONCE(tw->tw_substate), 0, 0,
                3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
                refcount_read(&tw->tw_refcnt), tw);
}

#define TMPSZ 150

static int tcp4_seq_show(struct seq_file *seq, void *v)
{
        struct tcp_iter_state *st;
        struct sock *sk = v;

        seq_setwidth(seq, TMPSZ - 1);
        if (v == SEQ_START_TOKEN) {
                seq_puts(seq, "  sl  local_address rem_address   st tx_queue "
                           "rx_queue tr tm->when retrnsmt   uid  timeout "
                           "inode");
                goto out;
        }
        st = seq->private;

        if (sk->sk_state == TCP_TIME_WAIT)
                get_timewait4_sock(v, seq, st->num);
        else if (sk->sk_state == TCP_NEW_SYN_RECV)
                get_openreq4(v, seq, st->num);
        else
                get_tcp4_sock(v, seq, st->num);
out:
        seq_pad(seq, '\n');
        return 0;
}

#ifdef CONFIG_BPF_SYSCALL
union bpf_tcp_iter_batch_item {
        struct sock *sk;
        __u64 cookie;
};

struct bpf_tcp_iter_state {
        struct tcp_iter_state state;
        unsigned int cur_sk;
        unsigned int end_sk;
        unsigned int max_sk;
        union bpf_tcp_iter_batch_item *batch;
};

struct bpf_iter__tcp {
        __bpf_md_ptr(struct bpf_iter_meta *, meta);
        __bpf_md_ptr(struct sock_common *, sk_common);
        uid_t uid __aligned(8);
};

static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
                             struct sock_common *sk_common, uid_t uid)
{
        struct bpf_iter__tcp ctx;

        meta->seq_num--;  /* skip SEQ_START_TOKEN */
        ctx.meta = meta;
        ctx.sk_common = sk_common;
        ctx.uid = uid;
        return bpf_iter_run_prog(prog, &ctx);
}

static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter)
{
        union bpf_tcp_iter_batch_item *item;
        unsigned int cur_sk = iter->cur_sk;
        __u64 cookie;

        /* Remember the cookies of the sockets we haven't seen yet, so we can
         * pick up where we left off next time around.
         */
        while (cur_sk < iter->end_sk) {
                item = &iter->batch[cur_sk++];
                cookie = sock_gen_cookie(item->sk);
                sock_gen_put(item->sk);
                item->cookie = cookie;
        }
}

static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
                                      unsigned int new_batch_sz, gfp_t flags)
{
        union bpf_tcp_iter_batch_item *new_batch;

        new_batch = kvmalloc(sizeof(*new_batch) * new_batch_sz,
                             flags | __GFP_NOWARN);
        if (!new_batch)
                return -ENOMEM;

        memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk);
        kvfree(iter->batch);
        iter->batch = new_batch;
        iter->max_sk = new_batch_sz;

        return 0;
}

static struct sock *bpf_iter_tcp_resume_bucket(struct sock *first_sk,
                                               union bpf_tcp_iter_batch_item *cookies,
                                               int n_cookies)
{
        struct hlist_nulls_node *node;
        struct sock *sk;
        int i;

        for (i = 0; i < n_cookies; i++) {
                sk = first_sk;
                sk_nulls_for_each_from(sk, node)
                        if (cookies[i].cookie == atomic64_read(&sk->sk_cookie))
                                return sk;
        }

        return NULL;
}

static struct sock *bpf_iter_tcp_resume_listening(struct seq_file *seq)
{
        struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
        struct bpf_tcp_iter_state *iter = seq->private;
        struct tcp_iter_state *st = &iter->state;
        unsigned int find_cookie = iter->cur_sk;
        unsigned int end_cookie = iter->end_sk;
        int resume_bucket = st->bucket;
        struct sock *sk;

        if (end_cookie && find_cookie == end_cookie)
                ++st->bucket;

        sk = listening_get_first(seq);
        iter->cur_sk = 0;
        iter->end_sk = 0;

        if (sk && st->bucket == resume_bucket && end_cookie) {
                sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
                                                end_cookie - find_cookie);
                if (!sk) {
                        spin_unlock(&hinfo->lhash2[st->bucket].lock);
                        ++st->bucket;
                        sk = listening_get_first(seq);
                }
        }

        return sk;
}

static struct sock *bpf_iter_tcp_resume_established(struct seq_file *seq)
{
        struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
        struct bpf_tcp_iter_state *iter = seq->private;
        struct tcp_iter_state *st = &iter->state;
        unsigned int find_cookie = iter->cur_sk;
        unsigned int end_cookie = iter->end_sk;
        int resume_bucket = st->bucket;
        struct sock *sk;

        if (end_cookie && find_cookie == end_cookie)
                ++st->bucket;

        sk = established_get_first(seq);
        iter->cur_sk = 0;
        iter->end_sk = 0;

        if (sk && st->bucket == resume_bucket && end_cookie) {
                sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
                                                end_cookie - find_cookie);
                if (!sk) {
                        spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
                        ++st->bucket;
                        sk = established_get_first(seq);
                }
        }

        return sk;
}

static struct sock *bpf_iter_tcp_resume(struct seq_file *seq)
{
        struct bpf_tcp_iter_state *iter = seq->private;
        struct tcp_iter_state *st = &iter->state;
        struct sock *sk = NULL;

        switch (st->state) {
        case TCP_SEQ_STATE_LISTENING:
                sk = bpf_iter_tcp_resume_listening(seq);
                if (sk)
                        break;
                st->bucket = 0;
                st->state = TCP_SEQ_STATE_ESTABLISHED;
                fallthrough;
        case TCP_SEQ_STATE_ESTABLISHED:
                sk = bpf_iter_tcp_resume_established(seq);
                break;
        }

        return sk;
}

static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
                                                 struct sock **start_sk)
{
        struct bpf_tcp_iter_state *iter = seq->private;
        struct hlist_nulls_node *node;
        unsigned int expected = 1;
        struct sock *sk;

        sock_hold(*start_sk);
        iter->batch[iter->end_sk++].sk = *start_sk;

        sk = sk_nulls_next(*start_sk);
        *start_sk = NULL;
        sk_nulls_for_each_from(sk, node) {
                if (seq_sk_match(seq, sk)) {
                        if (iter->end_sk < iter->max_sk) {
                                sock_hold(sk);
                                iter->batch[iter->end_sk++].sk = sk;
                        } else if (!*start_sk) {
                                /* Remember where we left off. */
                                *start_sk = sk;
                        }
                        expected++;
                }
        }

        return expected;
}

static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
                                                   struct sock **start_sk)
{
        struct bpf_tcp_iter_state *iter = seq->private;
        struct hlist_nulls_node *node;
        unsigned int expected = 1;
        struct sock *sk;

        sock_hold(*start_sk);
        iter->batch[iter->end_sk++].sk = *start_sk;

        sk = sk_nulls_next(*start_sk);
        *start_sk = NULL;
        sk_nulls_for_each_from(sk, node) {
                if (seq_sk_match(seq, sk)) {
                        if (iter->end_sk < iter->max_sk) {
                                sock_hold(sk);
                                iter->batch[iter->end_sk++].sk = sk;
                        } else if (!*start_sk) {
                                /* Remember where we left off. */
                                *start_sk = sk;
                        }
                        expected++;
                }
        }

        return expected;
}

static unsigned int bpf_iter_fill_batch(struct seq_file *seq,
                                        struct sock **start_sk)
{
        struct bpf_tcp_iter_state *iter = seq->private;
        struct tcp_iter_state *st = &iter->state;

        if (st->state == TCP_SEQ_STATE_LISTENING)
                return bpf_iter_tcp_listening_batch(seq, start_sk);
        else
                return bpf_iter_tcp_established_batch(seq, start_sk);
}

static void bpf_iter_tcp_unlock_bucket(struct seq_file *seq)
{
        struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
        struct bpf_tcp_iter_state *iter = seq->private;
        struct tcp_iter_state *st = &iter->state;

        if (st->state == TCP_SEQ_STATE_LISTENING)
                spin_unlock(&hinfo->lhash2[st->bucket].lock);
        else
                spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
}

static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
{
        struct bpf_tcp_iter_state *iter = seq->private;
        unsigned int expected;
        struct sock *sk;
        int err;

        sk = bpf_iter_tcp_resume(seq);
        if (!sk)
                return NULL; /* Done */

        expected = bpf_iter_fill_batch(seq, &sk);
        if (likely(iter->end_sk == expected))
                goto done;

        /* Batch size was too small. */
        bpf_iter_tcp_unlock_bucket(seq);
        bpf_iter_tcp_put_batch(iter);
        err = bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2,
                                         GFP_USER);
        if (err)
                return ERR_PTR(err);

        sk = bpf_iter_tcp_resume(seq);
        if (!sk)
                return NULL; /* Done */

        expected = bpf_iter_fill_batch(seq, &sk);
        if (likely(iter->end_sk == expected))
                goto done;

        /* Batch size was still too small. Hold onto the lock while we try
         * again with a larger batch to make sure the current bucket's size
         * does not change in the meantime.
         */
        err = bpf_iter_tcp_realloc_batch(iter, expected, GFP_NOWAIT);
        if (err) {
                bpf_iter_tcp_unlock_bucket(seq);
                return ERR_PTR(err);
        }

        expected = bpf_iter_fill_batch(seq, &sk);
        WARN_ON_ONCE(iter->end_sk != expected);
done:
        bpf_iter_tcp_unlock_bucket(seq);
        return iter->batch[0].sk;
}

static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos)
{
        /* bpf iter does not support lseek, so it always
         * continue from where it was stop()-ped.
         */
        if (*pos)
                return bpf_iter_tcp_batch(seq);

        return SEQ_START_TOKEN;
}

static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct bpf_tcp_iter_state *iter = seq->private;
        struct tcp_iter_state *st = &iter->state;
        struct sock *sk;

        /* Whenever seq_next() is called, the iter->cur_sk is
         * done with seq_show(), so advance to the next sk in
         * the batch.
         */
        if (iter->cur_sk < iter->end_sk) {
                /* Keeping st->num consistent in tcp_iter_state.
                 * bpf_iter_tcp does not use st->num.
                 * meta.seq_num is used instead.
                 */
                st->num++;
                sock_gen_put(iter->batch[iter->cur_sk++].sk);
        }

        if (iter->cur_sk < iter->end_sk)
                sk = iter->batch[iter->cur_sk].sk;
        else
                sk = bpf_iter_tcp_batch(seq);

        ++*pos;
        /* Keeping st->last_pos consistent in tcp_iter_state.
         * bpf iter does not do lseek, so st->last_pos always equals to *pos.
         */
        st->last_pos = *pos;
        return sk;
}

static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
{
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;
        struct sock *sk = v;
        uid_t uid;
        int ret;

        if (v == SEQ_START_TOKEN)
                return 0;

        if (sk_fullsock(sk))
                lock_sock(sk);

        if (unlikely(sk_unhashed(sk))) {
                ret = SEQ_SKIP;
                goto unlock;
        }

        if (sk->sk_state == TCP_TIME_WAIT) {
                uid = 0;
        } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
                const struct request_sock *req = v;

                uid = from_kuid_munged(seq_user_ns(seq),
                                       sk_uid(req->rsk_listener));
        } else {
                uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk));
        }

        meta.seq = seq;
        prog = bpf_iter_get_info(&meta, false);
        ret = tcp_prog_seq_show(prog, &meta, v, uid);

unlock:
        if (sk_fullsock(sk))
                release_sock(sk);
        return ret;

}

static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
{
        struct bpf_tcp_iter_state *iter = seq->private;
        struct bpf_iter_meta meta;
        struct bpf_prog *prog;

        if (!v) {
                meta.seq = seq;
                prog = bpf_iter_get_info(&meta, true);
                if (prog)
                        (void)tcp_prog_seq_show(prog, &meta, v, 0);
        }

        if (iter->cur_sk < iter->end_sk)
                bpf_iter_tcp_put_batch(iter);
}

static const struct seq_operations bpf_iter_tcp_seq_ops = {
        .show                = bpf_iter_tcp_seq_show,
        .start                = bpf_iter_tcp_seq_start,
        .next                = bpf_iter_tcp_seq_next,
        .stop                = bpf_iter_tcp_seq_stop,
};
#endif
static unsigned short seq_file_family(const struct seq_file *seq)
{
        const struct tcp_seq_afinfo *afinfo;

#ifdef CONFIG_BPF_SYSCALL
        /* Iterated from bpf_iter.  Let the bpf prog to filter instead. */
        if (seq->op == &bpf_iter_tcp_seq_ops)
                return AF_UNSPEC;
#endif

        /* Iterated from proc fs */
        afinfo = pde_data(file_inode(seq->file));
        return afinfo->family;
}

static const struct seq_operations tcp4_seq_ops = {
        .show                = tcp4_seq_show,
        .start                = tcp_seq_start,
        .next                = tcp_seq_next,
        .stop                = tcp_seq_stop,
};

static struct tcp_seq_afinfo tcp4_seq_afinfo = {
        .family                = AF_INET,
};

static int __net_init tcp4_proc_init_net(struct net *net)
{
        if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
                        sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
                return -ENOMEM;
        return 0;
}

static void __net_exit tcp4_proc_exit_net(struct net *net)
{
        remove_proc_entry("tcp", net->proc_net);
}

static struct pernet_operations tcp4_net_ops = {
        .init = tcp4_proc_init_net,
        .exit = tcp4_proc_exit_net,
};

int __init tcp4_proc_init(void)
{
        return register_pernet_subsys(&tcp4_net_ops);
}

void tcp4_proc_exit(void)
{
        unregister_pernet_subsys(&tcp4_net_ops);
}
#endif /* CONFIG_PROC_FS */

/* @wake is one when sk_stream_write_space() calls us.
 * This sends EPOLLOUT only if notsent_bytes is half the limit.
 * This mimics the strategy used in sock_def_write_space().
 */
bool tcp_stream_memory_free(const struct sock *sk, int wake)
{
        const struct tcp_sock *tp = tcp_sk(sk);
        u32 notsent_bytes = READ_ONCE(tp->write_seq) -
                            READ_ONCE(tp->snd_nxt);

        return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
}
EXPORT_SYMBOL(tcp_stream_memory_free);

struct proto tcp_prot = {
        .name                        = "TCP",
        .owner                        = THIS_MODULE,
        .close                        = tcp_close,
        .pre_connect                = tcp_v4_pre_connect,
        .connect                = tcp_v4_connect,
        .disconnect                = tcp_disconnect,
        .accept                        = inet_csk_accept,
        .ioctl                        = tcp_ioctl,
        .init                        = tcp_v4_init_sock,
        .destroy                = tcp_v4_destroy_sock,
        .shutdown                = tcp_shutdown,
        .setsockopt                = tcp_setsockopt,
        .getsockopt                = tcp_getsockopt,
        .bpf_bypass_getsockopt        = tcp_bpf_bypass_getsockopt,
        .keepalive                = tcp_set_keepalive,
        .recvmsg                = tcp_recvmsg,
        .sendmsg                = tcp_sendmsg,
        .splice_eof                = tcp_splice_eof,
        .backlog_rcv                = tcp_v4_do_rcv,
        .release_cb                = tcp_release_cb,
        .hash                        = inet_hash,
        .unhash                        = inet_unhash,
        .get_port                = inet_csk_get_port,
        .put_port                = inet_put_port,
#ifdef CONFIG_BPF_SYSCALL
        .psock_update_sk_prot        = tcp_bpf_update_proto,
#endif
        .enter_memory_pressure        = tcp_enter_memory_pressure,
        .leave_memory_pressure        = tcp_leave_memory_pressure,
        .stream_memory_free        = tcp_stream_memory_free,
        .sockets_allocated        = &tcp_sockets_allocated,

        .memory_allocated        = &net_aligned_data.tcp_memory_allocated,
        .per_cpu_fw_alloc        = &tcp_memory_per_cpu_fw_alloc,

        .memory_pressure        = &tcp_memory_pressure,
        .sysctl_mem                = sysctl_tcp_mem,
        .sysctl_wmem_offset        = offsetof(struct net, ipv4.sysctl_tcp_wmem),
        .sysctl_rmem_offset        = offsetof(struct net, ipv4.sysctl_tcp_rmem),
        .max_header                = MAX_TCP_HEADER,
        .obj_size                = sizeof(struct tcp_sock),
        .slab_flags                = SLAB_TYPESAFE_BY_RCU,
        .twsk_prot                = &tcp_timewait_sock_ops,
        .rsk_prot                = &tcp_request_sock_ops,
        .h.hashinfo                = NULL,
        .no_autobind                = true,
        .diag_destroy                = tcp_abort,
};
EXPORT_SYMBOL(tcp_prot);

static void __net_exit tcp_sk_exit(struct net *net)
{
        if (net->ipv4.tcp_congestion_control)
                bpf_module_put(net->ipv4.tcp_congestion_control,
                               net->ipv4.tcp_congestion_control->owner);
}

static void __net_init tcp_set_hashinfo(struct net *net)
{
        struct inet_hashinfo *hinfo;
        unsigned int ehash_entries;
        struct net *old_net;

        if (net_eq(net, &init_net))
                goto fallback;

        old_net = current->nsproxy->net_ns;
        ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries);
        if (!ehash_entries)
                goto fallback;

        ehash_entries = roundup_pow_of_two(ehash_entries);
        hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries);
        if (!hinfo) {
                pr_warn("Failed to allocate TCP ehash (entries: %u) "
                        "for a netns, fallback to the global one\n",
                        ehash_entries);
fallback:
                hinfo = &tcp_hashinfo;
                ehash_entries = tcp_hashinfo.ehash_mask + 1;
        }

        net->ipv4.tcp_death_row.hashinfo = hinfo;
        net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2;
        net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128);
}

static int __net_init tcp_sk_init(struct net *net)
{
        net->ipv4.sysctl_tcp_ecn = TCP_ECN_IN_ECN_OUT_NOECN;
        net->ipv4.sysctl_tcp_ecn_option = TCP_ACCECN_OPTION_FULL;
        net->ipv4.sysctl_tcp_ecn_option_beacon = TCP_ACCECN_OPTION_BEACON;
        net->ipv4.sysctl_tcp_ecn_fallback = 1;

        net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
        net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
        net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
        net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
        net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;

        net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
        net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
        net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;

        net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
        net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
        net->ipv4.sysctl_tcp_syncookies = 1;
        net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
        net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
        net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
        net->ipv4.sysctl_tcp_orphan_retries = 0;
        net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
        net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
        net->ipv4.sysctl_tcp_tw_reuse = 2;
        net->ipv4.sysctl_tcp_tw_reuse_delay = 1 * MSEC_PER_SEC;
        net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;

        refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
        tcp_set_hashinfo(net);

        net->ipv4.sysctl_tcp_sack = 1;
        net->ipv4.sysctl_tcp_window_scaling = 1;
        net->ipv4.sysctl_tcp_timestamps = 1;
        net->ipv4.sysctl_tcp_early_retrans = 3;
        net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
        net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior.  */
        net->ipv4.sysctl_tcp_retrans_collapse = 1;
        net->ipv4.sysctl_tcp_max_reordering = 300;
        net->ipv4.sysctl_tcp_dsack = 1;
        net->ipv4.sysctl_tcp_app_win = 31;
        net->ipv4.sysctl_tcp_adv_win_scale = 1;
        net->ipv4.sysctl_tcp_frto = 2;
        net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
        /* This limits the percentage of the congestion window which we
         * will allow a single TSO frame to consume.  Building TSO frames
         * which are too large can cause TCP streams to be bursty.
         */
        net->ipv4.sysctl_tcp_tso_win_divisor = 3;
        /* Default TSQ limit of 4 MB */
        net->ipv4.sysctl_tcp_limit_output_bytes = 4 << 20;

        /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */
        net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;

        net->ipv4.sysctl_tcp_min_tso_segs = 2;
        net->ipv4.sysctl_tcp_tso_rtt_log = 9;  /* 2^9 = 512 usec */
        net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
        net->ipv4.sysctl_tcp_autocorking = 1;
        net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
        net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
        net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
        if (net != &init_net) {
                memcpy(net->ipv4.sysctl_tcp_rmem,
                       init_net.ipv4.sysctl_tcp_rmem,
                       sizeof(init_net.ipv4.sysctl_tcp_rmem));
                memcpy(net->ipv4.sysctl_tcp_wmem,
                       init_net.ipv4.sysctl_tcp_wmem,
                       sizeof(init_net.ipv4.sysctl_tcp_wmem));
        }
        net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
        net->ipv4.sysctl_tcp_comp_sack_slack_ns = 100 * NSEC_PER_USEC;
        net->ipv4.sysctl_tcp_comp_sack_nr = 44;
        net->ipv4.sysctl_tcp_backlog_ack_defer = 1;
        net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
        net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
        atomic_set(&net->ipv4.tfo_active_disable_times, 0);

        /* Set default values for PLB */
        net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */
        net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3;
        net->ipv4.sysctl_tcp_plb_rehash_rounds = 12;
        net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60;
        /* Default congestion threshold for PLB to mark a round is 50% */
        net->ipv4.sysctl_tcp_plb_cong_thresh = (1 << TCP_PLB_SCALE) / 2;

        /* Reno is always built in */
        if (!net_eq(net, &init_net) &&
            bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
                               init_net.ipv4.tcp_congestion_control->owner))
                net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
        else
                net->ipv4.tcp_congestion_control = &tcp_reno;

        net->ipv4.sysctl_tcp_syn_linear_timeouts = 4;
        net->ipv4.sysctl_tcp_shrink_window = 0;

        net->ipv4.sysctl_tcp_pingpong_thresh = 1;
        net->ipv4.sysctl_tcp_rto_min_us = jiffies_to_usecs(TCP_RTO_MIN);
        net->ipv4.sysctl_tcp_rto_max_ms = TCP_RTO_MAX_SEC * MSEC_PER_SEC;

        return 0;
}

static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
{
        struct net *net;

        /* make sure concurrent calls to tcp_sk_exit_batch from net_cleanup_work
         * and failed setup_net error unwinding path are serialized.
         *
         * tcp_twsk_purge() handles twsk in any dead netns, not just those in
         * net_exit_list, the thread that dismantles a particular twsk must
         * do so without other thread progressing to refcount_dec_and_test() of
         * tcp_death_row.tw_refcount.
         */
        mutex_lock(&tcp_exit_batch_mutex);

        tcp_twsk_purge(net_exit_list);

        list_for_each_entry(net, net_exit_list, exit_list) {
                inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo);
                WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount));
                tcp_fastopen_ctx_destroy(net);
        }

        mutex_unlock(&tcp_exit_batch_mutex);
}

static struct pernet_operations __net_initdata tcp_sk_ops = {
       .init           = tcp_sk_init,
       .exit           = tcp_sk_exit,
       .exit_batch = tcp_sk_exit_batch,
};

#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
                     struct sock_common *sk_common, uid_t uid)

#define INIT_BATCH_SZ 16

static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
{
        struct bpf_tcp_iter_state *iter = priv_data;
        int err;

        err = bpf_iter_init_seq_net(priv_data, aux);
        if (err)
                return err;

        err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER);
        if (err) {
                bpf_iter_fini_seq_net(priv_data);
                return err;
        }

        return 0;
}

static void bpf_iter_fini_tcp(void *priv_data)
{
        struct bpf_tcp_iter_state *iter = priv_data;

        bpf_iter_fini_seq_net(priv_data);
        kvfree(iter->batch);
}

static const struct bpf_iter_seq_info tcp_seq_info = {
        .seq_ops                = &bpf_iter_tcp_seq_ops,
        .init_seq_private        = bpf_iter_init_tcp,
        .fini_seq_private        = bpf_iter_fini_tcp,
        .seq_priv_size                = sizeof(struct bpf_tcp_iter_state),
};

static const struct bpf_func_proto *
bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,
                            const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_setsockopt:
                return &bpf_sk_setsockopt_proto;
        case BPF_FUNC_getsockopt:
                return &bpf_sk_getsockopt_proto;
        default:
                return NULL;
        }
}

static struct bpf_iter_reg tcp_reg_info = {
        .target                        = "tcp",
        .ctx_arg_info_size        = 1,
        .ctx_arg_info                = {
                { offsetof(struct bpf_iter__tcp, sk_common),
                  PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
        },
        .get_func_proto                = bpf_iter_tcp_get_func_proto,
        .seq_info                = &tcp_seq_info,
};

static void __init bpf_iter_register(void)
{
        tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
        if (bpf_iter_reg_target(&tcp_reg_info))
                pr_warn("Warning: could not register bpf iterator tcp\n");
}

#endif

void __init tcp_v4_init(void)
{
        int cpu, res;

        for_each_possible_cpu(cpu) {
                struct sock *sk;

                res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
                                           IPPROTO_TCP, &init_net);
                if (res)
                        panic("Failed to create the TCP control socket.\n");
                sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);

                /* Please enforce IP_DF and IPID==0 for RST and
                 * ACK sent in SYN-RECV and TIME-WAIT state.
                 */
                inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;

                sk->sk_clockid = CLOCK_MONOTONIC;

                per_cpu(ipv4_tcp_sk.sock, cpu) = sk;
        }
        if (register_pernet_subsys(&tcp_sk_ops))
                panic("Failed to create the TCP control socket.\n");

#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
        bpf_iter_register();
#endif
}




























  208 











    5 








  217 


  208 





   12 


   12 












  215 

  217 


   14 










  217 

  217 


   14 


























  150 

  151 



  146 
















  193 

  187 





  151 




  139 




































  208 


  136 


  209 

















  209 




   13 








    1 

  210 






  210 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
// SPDX-License-Identifier: GPL-2.0
/*
 * ucs.c - Universal Character Set processing
 */

#include <linux/array_size.h>
#include <linux/bsearch.h>
#include <linux/consolemap.h>
#include <linux/minmax.h>

struct ucs_interval16 {
        u16 first;
        u16 last;
};

struct ucs_interval32 {
        u32 first;
        u32 last;
};

#include "ucs_width_table.h"

static int interval16_cmp(const void *key, const void *element)
{
        u16 cp = *(u16 *)key;
        const struct ucs_interval16 *entry = element;

        if (cp < entry->first)
                return -1;
        if (cp > entry->last)
                return 1;
        return 0;
}

static int interval32_cmp(const void *key, const void *element)
{
        u32 cp = *(u32 *)key;
        const struct ucs_interval32 *entry = element;

        if (cp < entry->first)
                return -1;
        if (cp > entry->last)
                return 1;
        return 0;
}

static bool cp_in_range16(u16 cp, const struct ucs_interval16 *ranges, size_t size)
{
        if (cp < ranges[0].first || cp > ranges[size - 1].last)
                return false;

        return __inline_bsearch(&cp, ranges, size, sizeof(*ranges),
                                interval16_cmp) != NULL;
}

static bool cp_in_range32(u32 cp, const struct ucs_interval32 *ranges, size_t size)
{
        if (cp < ranges[0].first || cp > ranges[size - 1].last)
                return false;

        return __inline_bsearch(&cp, ranges, size, sizeof(*ranges),
                                interval32_cmp) != NULL;
}

#define UCS_IS_BMP(cp)        ((cp) <= 0xffff)

/**
 * ucs_is_zero_width() - Determine if a Unicode code point is zero-width.
 * @cp: Unicode code point (UCS-4)
 *
 * Return: true if the character is zero-width, false otherwise
 */
bool ucs_is_zero_width(u32 cp)
{
        if (UCS_IS_BMP(cp))
                return cp_in_range16(cp, ucs_zero_width_bmp_ranges,
                                     ARRAY_SIZE(ucs_zero_width_bmp_ranges));
        else
                return cp_in_range32(cp, ucs_zero_width_non_bmp_ranges,
                                     ARRAY_SIZE(ucs_zero_width_non_bmp_ranges));
}

/**
 * ucs_is_double_width() - Determine if a Unicode code point is double-width.
 * @cp: Unicode code point (UCS-4)
 *
 * Return: true if the character is double-width, false otherwise
 */
bool ucs_is_double_width(u32 cp)
{
        if (UCS_IS_BMP(cp))
                return cp_in_range16(cp, ucs_double_width_bmp_ranges,
                                     ARRAY_SIZE(ucs_double_width_bmp_ranges));
        else
                return cp_in_range32(cp, ucs_double_width_non_bmp_ranges,
                                     ARRAY_SIZE(ucs_double_width_non_bmp_ranges));
}

/*
 * Structure for base with combining mark pairs and resulting recompositions.
 * Using u16 to save space since all values are within BMP range.
 */
struct ucs_recomposition {
        u16 base;        /* base character */
        u16 mark;        /* combining mark */
        u16 recomposed;        /* corresponding recomposed character */
};

#include "ucs_recompose_table.h"

struct compare_key {
        u16 base;
        u16 mark;
};

static int recomposition_cmp(const void *key, const void *element)
{
        const struct compare_key *search_key = key;
        const struct ucs_recomposition *entry = element;

        /* Compare base character first */
        if (search_key->base < entry->base)
                return -1;
        if (search_key->base > entry->base)
                return 1;

        /* Base characters match, now compare combining character */
        if (search_key->mark < entry->mark)
                return -1;
        if (search_key->mark > entry->mark)
                return 1;

        /* Both match */
        return 0;
}

/**
 * ucs_recompose() - Attempt to recompose two Unicode characters into a single character.
 * @base: Base Unicode code point (UCS-4)
 * @mark: Combining mark Unicode code point (UCS-4)
 *
 * Return: Recomposed Unicode code point, or 0 if no recomposition is possible
 */
u32 ucs_recompose(u32 base, u32 mark)
{
        /* Check if characters are within the range of our table */
        if (base < UCS_RECOMPOSE_MIN_BASE || base > UCS_RECOMPOSE_MAX_BASE ||
            mark < UCS_RECOMPOSE_MIN_MARK || mark > UCS_RECOMPOSE_MAX_MARK)
                return 0;

        struct compare_key key = { base, mark };
        struct ucs_recomposition *result =
                __inline_bsearch(&key, ucs_recomposition_table,
                                 ARRAY_SIZE(ucs_recomposition_table),
                                 sizeof(*ucs_recomposition_table),
                                 recomposition_cmp);

        return result ? result->recomposed : 0;
}

/*
 * The fallback table structures implement a 2-level lookup.
 */

struct ucs_page_desc {
        u8 page;        /* Page index (high byte of code points) */
        u8 count;        /* Number of entries in this page */
        u16 start;        /* Start index in entries array */
};

struct ucs_page_entry {
        u8 offset;        /* Offset within page (0-255) */
        u8 fallback;        /* Fallback character or range start marker */
};

#include "ucs_fallback_table.h"

static int ucs_page_desc_cmp(const void *key, const void *element)
{
        u8 page = *(u8 *)key;
        const struct ucs_page_desc *entry = element;

        if (page < entry->page)
                return -1;
        if (page > entry->page)
                return 1;
        return 0;
}

static int ucs_page_entry_cmp(const void *key, const void *element)
{
        u8 offset = *(u8 *)key;
        const struct ucs_page_entry *entry = element;

        if (offset < entry->offset)
                return -1;
        if (entry->fallback == UCS_PAGE_ENTRY_RANGE_MARKER) {
                if (offset > entry[1].offset)
                        return 1;
        } else {
                if (offset > entry->offset)
                        return 1;
        }
        return 0;
}

/**
 * ucs_get_fallback() - Get a substitution for the provided Unicode character
 * @cp: Unicode code point (UCS-4)
 *
 * Get a simpler fallback character for the provided Unicode character.
 * This is used for terminal display when corresponding glyph is unavailable.
 * The substitution may not be as good as the actual glyph for the original
 * character but still way more helpful than a squared question mark.
 *
 * Return: Fallback Unicode code point, or 0 if none is available
 */
u32 ucs_get_fallback(u32 cp)
{
        const struct ucs_page_desc *page;
        const struct ucs_page_entry *entry;
        u8 page_idx = cp >> 8, offset = cp;

        if (!UCS_IS_BMP(cp))
                return 0;

        /*
         * Full-width to ASCII mapping (covering all printable ASCII 33-126)
         * 0xFF01 (！) to 0xFF5E (～) -> ASCII 33 (!) to 126 (~)
         * We process them programmatically to reduce the table size.
         */
        if (cp >= 0xFF01 && cp <= 0xFF5E)
                return cp - 0xFF01 + 33;

        page = __inline_bsearch(&page_idx, ucs_fallback_pages,
                                ARRAY_SIZE(ucs_fallback_pages),
                                sizeof(*ucs_fallback_pages),
                                ucs_page_desc_cmp);
        if (!page)
                return 0;

        entry = __inline_bsearch(&offset, ucs_fallback_entries + page->start,
                                 page->count, sizeof(*ucs_fallback_entries),
                                 ucs_page_entry_cmp);
        if (!entry)
                return 0;

        if (entry->fallback == UCS_PAGE_ENTRY_RANGE_MARKER)
                entry++;
        return entry->fallback;
}












































 1181 
 1601 


















  624 















   90 

 2000 









































   23 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_POLL_H
#define _LINUX_POLL_H


#include <linux/compiler.h>
#include <linux/ktime.h>
#include <linux/wait.h>
#include <linux/string.h>
#include <linux/fs.h>
#include <linux/uaccess.h>
#include <uapi/linux/poll.h>
#include <uapi/linux/eventpoll.h>

/* ~832 bytes of stack space used max in sys_select/sys_poll before allocating
   additional memory. */
#define MAX_STACK_ALLOC 832
#define FRONTEND_STACK_ALLOC        256
#define SELECT_STACK_ALLOC        FRONTEND_STACK_ALLOC
#define POLL_STACK_ALLOC        FRONTEND_STACK_ALLOC
#define WQUEUES_STACK_ALLOC        (MAX_STACK_ALLOC - FRONTEND_STACK_ALLOC)
#define N_INLINE_POLL_ENTRIES        (WQUEUES_STACK_ALLOC / sizeof(struct poll_table_entry))

#define DEFAULT_POLLMASK (EPOLLIN | EPOLLOUT | EPOLLRDNORM | EPOLLWRNORM)

struct poll_table_struct;

/*
 * structures and helpers for f_op->poll implementations
 */
typedef void (*poll_queue_proc)(struct file *, wait_queue_head_t *, struct poll_table_struct *);

/*
 * Do not touch the structure directly, use the access function
 * poll_requested_events() instead.
 */
typedef struct poll_table_struct {
        poll_queue_proc _qproc;
        __poll_t _key;
} poll_table;

static inline void poll_wait(struct file * filp, wait_queue_head_t * wait_address, poll_table *p)
{
        if (p && p->_qproc) {
                p->_qproc(filp, wait_address, p);
                /*
                 * This memory barrier is paired in the wq_has_sleeper().
                 * See the comment above prepare_to_wait(), we need to
                 * ensure that subsequent tests in this thread can't be
                 * reordered with __add_wait_queue() in _qproc() paths.
                 */
                smp_mb();
        }
}

/*
 * Return the set of events that the application wants to poll for.
 * This is useful for drivers that need to know whether a DMA transfer has
 * to be started implicitly on poll(). You typically only want to do that
 * if the application is actually polling for POLLIN and/or POLLOUT.
 */
static inline __poll_t poll_requested_events(const poll_table *p)
{
        return p ? p->_key : ~(__poll_t)0;
}

static inline void init_poll_funcptr(poll_table *pt, poll_queue_proc qproc)
{
        pt->_qproc = qproc;
        pt->_key   = ~(__poll_t)0; /* all events enabled */
}

static inline bool file_can_poll(struct file *file)
{
        return file->f_op->poll;
}

static inline __poll_t vfs_poll(struct file *file, struct poll_table_struct *pt)
{
        if (unlikely(!file->f_op->poll))
                return DEFAULT_POLLMASK;
        return file->f_op->poll(file, pt);
}

struct poll_table_entry {
        struct file *filp;
        __poll_t key;
        wait_queue_entry_t wait;
        wait_queue_head_t *wait_address;
};

/*
 * Structures and helpers for select/poll syscall
 */
struct poll_wqueues {
        poll_table pt;
        struct poll_table_page *table;
        struct task_struct *polling_task;
        int triggered;
        int error;
        int inline_index;
        struct poll_table_entry inline_entries[N_INLINE_POLL_ENTRIES];
};

extern void poll_initwait(struct poll_wqueues *pwq);
extern void poll_freewait(struct poll_wqueues *pwq);
extern u64 select_estimate_accuracy(struct timespec64 *tv);

#define MAX_INT64_SECONDS (((s64)(~((u64)0)>>1)/HZ)-1)

extern int core_sys_select(int n, fd_set __user *inp, fd_set __user *outp,
                           fd_set __user *exp, struct timespec64 *end_time);

extern int poll_select_set_timeout(struct timespec64 *to, time64_t sec,
                                   long nsec);

#define __MAP(v, from, to) \
        (from < to ? (v & from) * (to/from) : (v & from) / (from/to))

static inline __u16 mangle_poll(__poll_t val)
{
        __u16 v = (__force __u16)val;
#define M(X) __MAP(v, (__force __u16)EPOLL##X, POLL##X)
        return M(IN) | M(OUT) | M(PRI) | M(ERR) | M(NVAL) |
                M(RDNORM) | M(RDBAND) | M(WRNORM) | M(WRBAND) |
                M(HUP) | M(RDHUP) | M(MSG);
#undef M
}

static inline __poll_t demangle_poll(u16 val)
{
#define M(X) (__force __poll_t)__MAP(val, POLL##X, (__force __u16)EPOLL##X)
        return M(IN) | M(OUT) | M(PRI) | M(ERR) | M(NVAL) |
                M(RDNORM) | M(RDBAND) | M(WRNORM) | M(WRBAND) |
                M(HUP) | M(RDHUP) | M(MSG);
#undef M
}
#undef __MAP


#endif /* _LINUX_POLL_H */























































































































































































































































































  244 

  242 




  243 



   11 









    2 



















































 1471 


 1472 




























































 1477 






   14 
  960 

























  233 


   10 






   25 
  214 











    5 
    5 






















   24 



   21 


    2 


   20 

    8 






    4 









  234 
    8 






   21 









   19 







   21 

   21 

    4 

























































   21 

   21 






    2 



   21 




















   18 
    3 







    4 
   18 





















    4 










  277 












   40 



  240 























  281 

    3 



  281 









  233 
































  233 


   32 




   61 

    6 


    2 












    6 







    7 


    8 


    4 


    6 









    4 















  234 






   16 


  222 


  234 

    5 






  234 








  234 









































  236 







  235 





  216 


   24 









  215 
   22 


  232 
    2 











  216 
   21 






   75 




 1378 

 1355 

   45 
   45 
























   10 









    2 


    2 


   17 





   10 















   17 




























   10 




   10 




































   19 











   13 

    3 

   11 


    8 


    2 


    4 















   11 




















   19 

   19 






   19 







   25 
    2 





   17 




   11 






    7 
   14 

   12 

    8 





   16 
















    2 
   16 

    1 


   16 








 1469 













 1478 



  540 


    2 




















    2 



































   29 





  396 























    5 







   21 












 1478 











   10 


 1467 












    1 

















    1 





   10 
















    3 


    3 








   18 



    5 
















   18 


   18 
   15 






    7 


   17 

    7 
    3 








   10 

    1 

   10 







    1 


   11 





    3 







    3 













    1 


    2 




   12 




    2 
    1 

























    6 









    3 


    2 
    1 

















    1 


    1 


    6 
    1 






    5 



















    2 












    1 






    3 
    1 













    1 





    1 
















    1 


    9 



    1 


    1 


    1 



    1 













    2 














    3 













    2 


    5 













    5 


























   46 




    4 

   15 

    9 


    4 

    2 




    3 

   13 







   22 




   22 




   22 




   10 



















   36 
















   36 

















  239 



  239 

  205 

   34 
    3 





























    1 









    1 













  284 
   40 


  206 


    1 




   34 










  239 








    3 





    4 




  234 


  218 
   25 

  206 
    3 





   32 




  240 














    2 


   10 


















    1 














  312 
    1 

    1 

   11 

  286 

    1 


   11 

    1 


    1 





  236 













   79 



   11 












   68 

























































    1 




    1 
































    1 




    1 























































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
// SPDX-License-Identifier: GPL-2.0
/*
 * linux/kernel/seccomp.c
 *
 * Copyright 2004-2005  Andrea Arcangeli <andrea@cpushare.com>
 *
 * Copyright (C) 2012 Google, Inc.
 * Will Drewry <wad@chromium.org>
 *
 * This defines a simple but solid secure-computing facility.
 *
 * Mode 1 uses a fixed list of allowed system calls.
 * Mode 2 allows user-defined system call filters in the form
 *        of Berkeley Packet Filters/Linux Socket Filters.
 */
#define pr_fmt(fmt) "seccomp: " fmt

#include <linux/refcount.h>
#include <linux/audit.h>
#include <linux/compat.h>
#include <linux/coredump.h>
#include <linux/kmemleak.h>
#include <linux/nospec.h>
#include <linux/prctl.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/seccomp.h>
#include <linux/slab.h>
#include <linux/syscalls.h>
#include <linux/sysctl.h>

#include <asm/syscall.h>

/* Not exposed in headers: strictly internal use only. */
#define SECCOMP_MODE_DEAD        (SECCOMP_MODE_FILTER + 1)

#ifdef CONFIG_SECCOMP_FILTER
#include <linux/file.h>
#include <linux/filter.h>
#include <linux/pid.h>
#include <linux/ptrace.h>
#include <linux/capability.h>
#include <linux/uaccess.h>
#include <linux/anon_inodes.h>
#include <linux/lockdep.h>

/*
 * When SECCOMP_IOCTL_NOTIF_ID_VALID was first introduced, it had the
 * wrong direction flag in the ioctl number. This is the broken one,
 * which the kernel needs to keep supporting until all userspaces stop
 * using the wrong command number.
 */
#define SECCOMP_IOCTL_NOTIF_ID_VALID_WRONG_DIR        SECCOMP_IOR(2, __u64)

enum notify_state {
        SECCOMP_NOTIFY_INIT,
        SECCOMP_NOTIFY_SENT,
        SECCOMP_NOTIFY_REPLIED,
};

struct seccomp_knotif {
        /* The struct pid of the task whose filter triggered the notification */
        struct task_struct *task;

        /* The "cookie" for this request; this is unique for this filter. */
        u64 id;

        /*
         * The seccomp data. This pointer is valid the entire time this
         * notification is active, since it comes from __seccomp_filter which
         * eclipses the entire lifecycle here.
         */
        const struct seccomp_data *data;

        /*
         * Notification states. When SECCOMP_RET_USER_NOTIF is returned, a
         * struct seccomp_knotif is created and starts out in INIT. Once the
         * handler reads the notification off of an FD, it transitions to SENT.
         * If a signal is received the state transitions back to INIT and
         * another message is sent. When the userspace handler replies, state
         * transitions to REPLIED.
         */
        enum notify_state state;

        /* The return values, only valid when in SECCOMP_NOTIFY_REPLIED */
        int error;
        long val;
        u32 flags;

        /*
         * Signals when this has changed states, such as the listener
         * dying, a new seccomp addfd message, or changing to REPLIED
         */
        struct completion ready;

        struct list_head list;

        /* outstanding addfd requests */
        struct list_head addfd;
};

/**
 * struct seccomp_kaddfd - container for seccomp_addfd ioctl messages
 *
 * @file: A reference to the file to install in the other task
 * @fd: The fd number to install it at. If the fd number is -1, it means the
 *      installing process should allocate the fd as normal.
 * @flags: The flags for the new file descriptor. At the moment, only O_CLOEXEC
 *         is allowed.
 * @ioctl_flags: The flags used for the seccomp_addfd ioctl.
 * @setfd: whether or not SECCOMP_ADDFD_FLAG_SETFD was set during notify_addfd
 * @ret: The return value of the installing process. It is set to the fd num
 *       upon success (>= 0).
 * @completion: Indicates that the installing process has completed fd
 *              installation, or gone away (either due to successful
 *              reply, or signal)
 * @list: list_head for chaining seccomp_kaddfd together.
 *
 */
struct seccomp_kaddfd {
        struct file *file;
        int fd;
        unsigned int flags;
        __u32 ioctl_flags;

        union {
                bool setfd;
                /* To only be set on reply */
                int ret;
        };
        struct completion completion;
        struct list_head list;
};

/**
 * struct notification - container for seccomp userspace notifications. Since
 * most seccomp filters will not have notification listeners attached and this
 * structure is fairly large, we store the notification-specific stuff in a
 * separate structure.
 *
 * @requests: A semaphore that users of this notification can wait on for
 *            changes. Actual reads and writes are still controlled with
 *            filter->notify_lock.
 * @flags: A set of SECCOMP_USER_NOTIF_FD_* flags.
 * @next_id: The id of the next request.
 * @notifications: A list of struct seccomp_knotif elements.
 */

struct notification {
        atomic_t requests;
        u32 flags;
        u64 next_id;
        struct list_head notifications;
};

#ifdef SECCOMP_ARCH_NATIVE
/**
 * struct action_cache - per-filter cache of seccomp actions per
 * arch/syscall pair
 *
 * @allow_native: A bitmap where each bit represents whether the
 *                  filter will always allow the syscall, for the
 *                  native architecture.
 * @allow_compat: A bitmap where each bit represents whether the
 *                  filter will always allow the syscall, for the
 *                  compat architecture.
 */
struct action_cache {
        DECLARE_BITMAP(allow_native, SECCOMP_ARCH_NATIVE_NR);
#ifdef SECCOMP_ARCH_COMPAT
        DECLARE_BITMAP(allow_compat, SECCOMP_ARCH_COMPAT_NR);
#endif
};
#else
struct action_cache { };

static inline bool seccomp_cache_check_allow(const struct seccomp_filter *sfilter,
                                             const struct seccomp_data *sd)
{
        return false;
}

static inline void seccomp_cache_prepare(struct seccomp_filter *sfilter)
{
}
#endif /* SECCOMP_ARCH_NATIVE */

/**
 * struct seccomp_filter - container for seccomp BPF programs
 *
 * @refs: Reference count to manage the object lifetime.
 *          A filter's reference count is incremented for each directly
 *          attached task, once for the dependent filter, and if
 *          requested for the user notifier. When @refs reaches zero,
 *          the filter can be freed.
 * @users: A filter's @users count is incremented for each directly
 *         attached task (filter installation, fork(), thread_sync),
 *           and once for the dependent filter (tracked in filter->prev).
 *           When it reaches zero it indicates that no direct or indirect
 *           users of that filter exist. No new tasks can get associated with
 *           this filter after reaching 0. The @users count is always smaller
 *           or equal to @refs. Hence, reaching 0 for @users does not mean
 *           the filter can be freed.
 * @cache: cache of arch/syscall mappings to actions
 * @log: true if all actions except for SECCOMP_RET_ALLOW should be logged
 * @wait_killable_recv: Put notifying process in killable state once the
 *                        notification is received by the userspace listener.
 * @prev: points to a previously installed, or inherited, filter
 * @prog: the BPF program to evaluate
 * @notif: the struct that holds all notification related information
 * @notify_lock: A lock for all notification-related accesses.
 * @wqh: A wait queue for poll if a notifier is in use.
 *
 * seccomp_filter objects are organized in a tree linked via the @prev
 * pointer.  For any task, it appears to be a singly-linked list starting
 * with current->seccomp.filter, the most recently attached or inherited filter.
 * However, multiple filters may share a @prev node, by way of fork(), which
 * results in a unidirectional tree existing in memory.  This is similar to
 * how namespaces work.
 *
 * seccomp_filter objects should never be modified after being attached
 * to a task_struct (other than @refs).
 */
struct seccomp_filter {
        refcount_t refs;
        refcount_t users;
        bool log;
        bool wait_killable_recv;
        struct action_cache cache;
        struct seccomp_filter *prev;
        struct bpf_prog *prog;
        struct notification *notif;
        struct mutex notify_lock;
        wait_queue_head_t wqh;
};

/* Limit any path through the tree to 256KB worth of instructions. */
#define MAX_INSNS_PER_PATH ((1 << 18) / sizeof(struct sock_filter))

/*
 * Endianness is explicitly ignored and left for BPF program authors to manage
 * as per the specific architecture.
 */
static void populate_seccomp_data(struct seccomp_data *sd)
{
        /*
         * Instead of using current_pt_reg(), we're already doing the work
         * to safely fetch "current", so just use "task" everywhere below.
         */
        struct task_struct *task = current;
        struct pt_regs *regs = task_pt_regs(task);
        unsigned long args[6];

        sd->nr = syscall_get_nr(task, regs);
        sd->arch = syscall_get_arch(task);
        syscall_get_arguments(task, regs, args);
        sd->args[0] = args[0];
        sd->args[1] = args[1];
        sd->args[2] = args[2];
        sd->args[3] = args[3];
        sd->args[4] = args[4];
        sd->args[5] = args[5];
        sd->instruction_pointer = KSTK_EIP(task);
}

/**
 *        seccomp_check_filter - verify seccomp filter code
 *        @filter: filter to verify
 *        @flen: length of filter
 *
 * Takes a previously checked filter (by bpf_check_classic) and
 * redirects all filter code that loads struct sk_buff data
 * and related data through seccomp_bpf_load.  It also
 * enforces length and alignment checking of those loads.
 *
 * Returns 0 if the rule set is legal or -EINVAL if not.
 */
static int seccomp_check_filter(struct sock_filter *filter, unsigned int flen)
{
        int pc;
        for (pc = 0; pc < flen; pc++) {
                struct sock_filter *ftest = &filter[pc];
                u16 code = ftest->code;
                u32 k = ftest->k;

                switch (code) {
                case BPF_LD | BPF_W | BPF_ABS:
                        ftest->code = BPF_LDX | BPF_W | BPF_ABS;
                        /* 32-bit aligned and not out of bounds. */
                        if (k >= sizeof(struct seccomp_data) || k & 3)
                                return -EINVAL;
                        continue;
                case BPF_LD | BPF_W | BPF_LEN:
                        ftest->code = BPF_LD | BPF_IMM;
                        ftest->k = sizeof(struct seccomp_data);
                        continue;
                case BPF_LDX | BPF_W | BPF_LEN:
                        ftest->code = BPF_LDX | BPF_IMM;
                        ftest->k = sizeof(struct seccomp_data);
                        continue;
                /* Explicitly include allowed calls. */
                case BPF_RET | BPF_K:
                case BPF_RET | BPF_A:
                case BPF_ALU | BPF_ADD | BPF_K:
                case BPF_ALU | BPF_ADD | BPF_X:
                case BPF_ALU | BPF_SUB | BPF_K:
                case BPF_ALU | BPF_SUB | BPF_X:
                case BPF_ALU | BPF_MUL | BPF_K:
                case BPF_ALU | BPF_MUL | BPF_X:
                case BPF_ALU | BPF_DIV | BPF_K:
                case BPF_ALU | BPF_DIV | BPF_X:
                case BPF_ALU | BPF_AND | BPF_K:
                case BPF_ALU | BPF_AND | BPF_X:
                case BPF_ALU | BPF_OR | BPF_K:
                case BPF_ALU | BPF_OR | BPF_X:
                case BPF_ALU | BPF_XOR | BPF_K:
                case BPF_ALU | BPF_XOR | BPF_X:
                case BPF_ALU | BPF_LSH | BPF_K:
                case BPF_ALU | BPF_LSH | BPF_X:
                case BPF_ALU | BPF_RSH | BPF_K:
                case BPF_ALU | BPF_RSH | BPF_X:
                case BPF_ALU | BPF_NEG:
                case BPF_LD | BPF_IMM:
                case BPF_LDX | BPF_IMM:
                case BPF_MISC | BPF_TAX:
                case BPF_MISC | BPF_TXA:
                case BPF_LD | BPF_MEM:
                case BPF_LDX | BPF_MEM:
                case BPF_ST:
                case BPF_STX:
                case BPF_JMP | BPF_JA:
                case BPF_JMP | BPF_JEQ | BPF_K:
                case BPF_JMP | BPF_JEQ | BPF_X:
                case BPF_JMP | BPF_JGE | BPF_K:
                case BPF_JMP | BPF_JGE | BPF_X:
                case BPF_JMP | BPF_JGT | BPF_K:
                case BPF_JMP | BPF_JGT | BPF_X:
                case BPF_JMP | BPF_JSET | BPF_K:
                case BPF_JMP | BPF_JSET | BPF_X:
                        continue;
                default:
                        return -EINVAL;
                }
        }
        return 0;
}

#ifdef SECCOMP_ARCH_NATIVE
static inline bool seccomp_cache_check_allow_bitmap(const void *bitmap,
                                                    size_t bitmap_size,
                                                    int syscall_nr)
{
        if (unlikely(syscall_nr < 0 || syscall_nr >= bitmap_size))
                return false;
        syscall_nr = array_index_nospec(syscall_nr, bitmap_size);

        return test_bit(syscall_nr, bitmap);
}

/**
 * seccomp_cache_check_allow - lookup seccomp cache
 * @sfilter: The seccomp filter
 * @sd: The seccomp data to lookup the cache with
 *
 * Returns true if the seccomp_data is cached and allowed.
 */
static inline bool seccomp_cache_check_allow(const struct seccomp_filter *sfilter,
                                             const struct seccomp_data *sd)
{
        int syscall_nr = sd->nr;
        const struct action_cache *cache = &sfilter->cache;

#ifndef SECCOMP_ARCH_COMPAT
        /* A native-only architecture doesn't need to check sd->arch. */
        return seccomp_cache_check_allow_bitmap(cache->allow_native,
                                                SECCOMP_ARCH_NATIVE_NR,
                                                syscall_nr);
#else
        if (likely(sd->arch == SECCOMP_ARCH_NATIVE))
                return seccomp_cache_check_allow_bitmap(cache->allow_native,
                                                        SECCOMP_ARCH_NATIVE_NR,
                                                        syscall_nr);
        if (likely(sd->arch == SECCOMP_ARCH_COMPAT))
                return seccomp_cache_check_allow_bitmap(cache->allow_compat,
                                                        SECCOMP_ARCH_COMPAT_NR,
                                                        syscall_nr);
#endif /* SECCOMP_ARCH_COMPAT */

        WARN_ON_ONCE(true);
        return false;
}
#endif /* SECCOMP_ARCH_NATIVE */

#define ACTION_ONLY(ret) ((s32)((ret) & (SECCOMP_RET_ACTION_FULL)))
/**
 * seccomp_run_filters - evaluates all seccomp filters against @sd
 * @sd: optional seccomp data to be passed to filters
 * @match: stores struct seccomp_filter that resulted in the return value,
 *         unless filter returned SECCOMP_RET_ALLOW, in which case it will
 *         be unchanged.
 *
 * Returns valid seccomp BPF response codes.
 */
static u32 seccomp_run_filters(const struct seccomp_data *sd,
                               struct seccomp_filter **match)
{
        u32 ret = SECCOMP_RET_ALLOW;
        /* Make sure cross-thread synced filter points somewhere sane. */
        struct seccomp_filter *f =
                        READ_ONCE(current->seccomp.filter);

        /* Ensure unexpected behavior doesn't result in failing open. */
        if (WARN_ON(f == NULL))
                return SECCOMP_RET_KILL_PROCESS;

        if (seccomp_cache_check_allow(f, sd))
                return SECCOMP_RET_ALLOW;

        /*
         * All filters in the list are evaluated and the lowest BPF return
         * value always takes priority (ignoring the DATA).
         */
        for (; f; f = f->prev) {
                u32 cur_ret = bpf_prog_run_pin_on_cpu(f->prog, sd);

                if (ACTION_ONLY(cur_ret) < ACTION_ONLY(ret)) {
                        ret = cur_ret;
                        *match = f;
                }
        }
        return ret;
}
#endif /* CONFIG_SECCOMP_FILTER */

static inline bool seccomp_may_assign_mode(unsigned long seccomp_mode)
{
        assert_spin_locked(&current->sighand->siglock);

        if (current->seccomp.mode && current->seccomp.mode != seccomp_mode)
                return false;

        return true;
}

void __weak arch_seccomp_spec_mitigate(struct task_struct *task) { }

static inline void seccomp_assign_mode(struct task_struct *task,
                                       unsigned long seccomp_mode,
                                       unsigned long flags)
{
        assert_spin_locked(&task->sighand->siglock);

        task->seccomp.mode = seccomp_mode;
        /*
         * Make sure SYSCALL_WORK_SECCOMP cannot be set before the mode (and
         * filter) is set.
         */
        smp_mb__before_atomic();
        /* Assume default seccomp processes want spec flaw mitigation. */
        if ((flags & SECCOMP_FILTER_FLAG_SPEC_ALLOW) == 0)
                arch_seccomp_spec_mitigate(task);
        set_task_syscall_work(task, SECCOMP);
}

#ifdef CONFIG_SECCOMP_FILTER
/* Returns 1 if the parent is an ancestor of the child. */
static int is_ancestor(struct seccomp_filter *parent,
                       struct seccomp_filter *child)
{
        /* NULL is the root ancestor. */
        if (parent == NULL)
                return 1;
        for (; child; child = child->prev)
                if (child == parent)
                        return 1;
        return 0;
}

/**
 * seccomp_can_sync_threads: checks if all threads can be synchronized
 *
 * Expects sighand and cred_guard_mutex locks to be held.
 *
 * Returns 0 on success, -ve on error, or the pid of a thread which was
 * either not in the correct seccomp mode or did not have an ancestral
 * seccomp filter.
 */
static inline pid_t seccomp_can_sync_threads(void)
{
        struct task_struct *thread, *caller;

        BUG_ON(!mutex_is_locked(&current->signal->cred_guard_mutex));
        assert_spin_locked(&current->sighand->siglock);

        /* Validate all threads being eligible for synchronization. */
        caller = current;
        for_each_thread(caller, thread) {
                pid_t failed;

                /* Skip current, since it is initiating the sync. */
                if (thread == caller)
                        continue;
                /* Skip exited threads. */
                if (thread->flags & PF_EXITING)
                        continue;

                if (thread->seccomp.mode == SECCOMP_MODE_DISABLED ||
                    (thread->seccomp.mode == SECCOMP_MODE_FILTER &&
                     is_ancestor(thread->seccomp.filter,
                                 caller->seccomp.filter)))
                        continue;

                /* Return the first thread that cannot be synchronized. */
                failed = task_pid_vnr(thread);
                /* If the pid cannot be resolved, then return -ESRCH */
                if (WARN_ON(failed == 0))
                        failed = -ESRCH;
                return failed;
        }

        return 0;
}

static inline void seccomp_filter_free(struct seccomp_filter *filter)
{
        if (filter) {
                bpf_prog_destroy(filter->prog);
                kfree(filter);
        }
}

static void __seccomp_filter_orphan(struct seccomp_filter *orig)
{
        while (orig && refcount_dec_and_test(&orig->users)) {
                if (waitqueue_active(&orig->wqh))
                        wake_up_poll(&orig->wqh, EPOLLHUP);
                orig = orig->prev;
        }
}

static void __put_seccomp_filter(struct seccomp_filter *orig)
{
        /* Clean up single-reference branches iteratively. */
        while (orig && refcount_dec_and_test(&orig->refs)) {
                struct seccomp_filter *freeme = orig;
                orig = orig->prev;
                seccomp_filter_free(freeme);
        }
}

static void __seccomp_filter_release(struct seccomp_filter *orig)
{
        /* Notify about any unused filters in the task's former filter tree. */
        __seccomp_filter_orphan(orig);
        /* Finally drop all references to the task's former tree. */
        __put_seccomp_filter(orig);
}

/**
 * seccomp_filter_release - Detach the task from its filter tree,
 *                            drop its reference count, and notify
 *                            about unused filters
 *
 * @tsk: task the filter should be released from.
 *
 * This function should only be called when the task is exiting as
 * it detaches it from its filter tree. PF_EXITING has to be set
 * for the task.
 */
void seccomp_filter_release(struct task_struct *tsk)
{
        struct seccomp_filter *orig;

        if (WARN_ON((tsk->flags & PF_EXITING) == 0))
                return;

        if (READ_ONCE(tsk->seccomp.filter) == NULL)
                return;

        spin_lock_irq(&tsk->sighand->siglock);
        orig = tsk->seccomp.filter;
        /* Detach task from its filter tree. */
        tsk->seccomp.filter = NULL;
        spin_unlock_irq(&tsk->sighand->siglock);
        __seccomp_filter_release(orig);
}

/**
 * seccomp_sync_threads: sets all threads to use current's filter
 *
 * @flags: SECCOMP_FILTER_FLAG_* flags to set during sync.
 *
 * Expects sighand and cred_guard_mutex locks to be held, and for
 * seccomp_can_sync_threads() to have returned success already
 * without dropping the locks.
 *
 */
static inline void seccomp_sync_threads(unsigned long flags)
{
        struct task_struct *thread, *caller;

        BUG_ON(!mutex_is_locked(&current->signal->cred_guard_mutex));
        assert_spin_locked(&current->sighand->siglock);

        /*
         * Don't touch any of the threads if the process is being killed.
         * This allows for a lockless check in seccomp_filter_release.
         */
        if (current->signal->flags & SIGNAL_GROUP_EXIT)
                return;

        /* Synchronize all threads. */
        caller = current;
        for_each_thread(caller, thread) {
                /* Skip current, since it needs no changes. */
                if (thread == caller)
                        continue;

                /*
                 * Skip exited threads. seccomp_filter_release could have
                 * been already called for this task.
                 */
                if (thread->flags & PF_EXITING)
                        continue;

                /* Get a task reference for the new leaf node. */
                get_seccomp_filter(caller);

                /*
                 * Drop the task reference to the shared ancestor since
                 * current's path will hold a reference.  (This also
                 * allows a put before the assignment.)
                 */
                __seccomp_filter_release(thread->seccomp.filter);

                /* Make our new filter tree visible. */
                smp_store_release(&thread->seccomp.filter,
                                  caller->seccomp.filter);
                atomic_set(&thread->seccomp.filter_count,
                           atomic_read(&caller->seccomp.filter_count));

                /*
                 * Don't let an unprivileged task work around
                 * the no_new_privs restriction by creating
                 * a thread that sets it up, enters seccomp,
                 * then dies.
                 */
                if (task_no_new_privs(caller))
                        task_set_no_new_privs(thread);

                /*
                 * Opt the other thread into seccomp if needed.
                 * As threads are considered to be trust-realm
                 * equivalent (see ptrace_may_access), it is safe to
                 * allow one thread to transition the other.
                 */
                if (thread->seccomp.mode == SECCOMP_MODE_DISABLED)
                        seccomp_assign_mode(thread, SECCOMP_MODE_FILTER,
                                            flags);
        }
}

/**
 * seccomp_prepare_filter: Prepares a seccomp filter for use.
 * @fprog: BPF program to install
 *
 * Returns filter on success or an ERR_PTR on failure.
 */
static struct seccomp_filter *seccomp_prepare_filter(struct sock_fprog *fprog)
{
        struct seccomp_filter *sfilter;
        int ret;
        const bool save_orig =
#if defined(CONFIG_CHECKPOINT_RESTORE) || defined(SECCOMP_ARCH_NATIVE)
                true;
#else
                false;
#endif

        if (fprog->len == 0 || fprog->len > BPF_MAXINSNS)
                return ERR_PTR(-EINVAL);

        BUG_ON(INT_MAX / fprog->len < sizeof(struct sock_filter));

        /*
         * Installing a seccomp filter requires that the task has
         * CAP_SYS_ADMIN in its namespace or be running with no_new_privs.
         * This avoids scenarios where unprivileged tasks can affect the
         * behavior of privileged children.
         */
        if (!task_no_new_privs(current) &&
                        !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN))
                return ERR_PTR(-EACCES);

        /* Allocate a new seccomp_filter */
        sfilter = kzalloc(sizeof(*sfilter), GFP_KERNEL | __GFP_NOWARN);
        if (!sfilter)
                return ERR_PTR(-ENOMEM);

        mutex_init(&sfilter->notify_lock);
        ret = bpf_prog_create_from_user(&sfilter->prog, fprog,
                                        seccomp_check_filter, save_orig);
        if (ret < 0) {
                kfree(sfilter);
                return ERR_PTR(ret);
        }

        refcount_set(&sfilter->refs, 1);
        refcount_set(&sfilter->users, 1);
        init_waitqueue_head(&sfilter->wqh);

        return sfilter;
}

/**
 * seccomp_prepare_user_filter - prepares a user-supplied sock_fprog
 * @user_filter: pointer to the user data containing a sock_fprog.
 *
 * Returns 0 on success and non-zero otherwise.
 */
static struct seccomp_filter *
seccomp_prepare_user_filter(const char __user *user_filter)
{
        struct sock_fprog fprog;
        struct seccomp_filter *filter = ERR_PTR(-EFAULT);

#ifdef CONFIG_COMPAT
        if (in_compat_syscall()) {
                struct compat_sock_fprog fprog32;
                if (copy_from_user(&fprog32, user_filter, sizeof(fprog32)))
                        goto out;
                fprog.len = fprog32.len;
                fprog.filter = compat_ptr(fprog32.filter);
        } else /* falls through to the if below. */
#endif
        if (copy_from_user(&fprog, user_filter, sizeof(fprog)))
                goto out;
        filter = seccomp_prepare_filter(&fprog);
out:
        return filter;
}

#ifdef SECCOMP_ARCH_NATIVE
static bool seccomp_uprobe_exception(struct seccomp_data *sd)
{
#if defined __NR_uretprobe || defined __NR_uprobe
#ifdef SECCOMP_ARCH_COMPAT
        if (sd->arch == SECCOMP_ARCH_NATIVE)
#endif
        {
#ifdef __NR_uretprobe
                if (sd->nr == __NR_uretprobe)
                        return true;
#endif
#ifdef __NR_uprobe
                if (sd->nr == __NR_uprobe)
                        return true;
#endif
        }
#endif
        return false;
}

/**
 * seccomp_is_const_allow - check if filter is constant allow with given data
 * @fprog: The BPF programs
 * @sd: The seccomp data to check against, only syscall number and arch
 *      number are considered constant.
 */
static bool seccomp_is_const_allow(struct sock_fprog_kern *fprog,
                                   struct seccomp_data *sd)
{
        unsigned int reg_value = 0;
        unsigned int pc;
        bool op_res;

        if (WARN_ON_ONCE(!fprog))
                return false;

        /* Our single exception to filtering. */
        if (seccomp_uprobe_exception(sd))
                return true;

        for (pc = 0; pc < fprog->len; pc++) {
                struct sock_filter *insn = &fprog->filter[pc];
                u16 code = insn->code;
                u32 k = insn->k;

                switch (code) {
                case BPF_LD | BPF_W | BPF_ABS:
                        switch (k) {
                        case offsetof(struct seccomp_data, nr):
                                reg_value = sd->nr;
                                break;
                        case offsetof(struct seccomp_data, arch):
                                reg_value = sd->arch;
                                break;
                        default:
                                /* can't optimize (non-constant value load) */
                                return false;
                        }
                        break;
                case BPF_RET | BPF_K:
                        /* reached return with constant values only, check allow */
                        return k == SECCOMP_RET_ALLOW;
                case BPF_JMP | BPF_JA:
                        pc += insn->k;
                        break;
                case BPF_JMP | BPF_JEQ | BPF_K:
                case BPF_JMP | BPF_JGE | BPF_K:
                case BPF_JMP | BPF_JGT | BPF_K:
                case BPF_JMP | BPF_JSET | BPF_K:
                        switch (BPF_OP(code)) {
                        case BPF_JEQ:
                                op_res = reg_value == k;
                                break;
                        case BPF_JGE:
                                op_res = reg_value >= k;
                                break;
                        case BPF_JGT:
                                op_res = reg_value > k;
                                break;
                        case BPF_JSET:
                                op_res = !!(reg_value & k);
                                break;
                        default:
                                /* can't optimize (unknown jump) */
                                return false;
                        }

                        pc += op_res ? insn->jt : insn->jf;
                        break;
                case BPF_ALU | BPF_AND | BPF_K:
                        reg_value &= k;
                        break;
                default:
                        /* can't optimize (unknown insn) */
                        return false;
                }
        }

        /* ran off the end of the filter?! */
        WARN_ON(1);
        return false;
}

static void seccomp_cache_prepare_bitmap(struct seccomp_filter *sfilter,
                                         void *bitmap, const void *bitmap_prev,
                                         size_t bitmap_size, int arch)
{
        struct sock_fprog_kern *fprog = sfilter->prog->orig_prog;
        struct seccomp_data sd;
        int nr;

        if (bitmap_prev) {
                /* The new filter must be as restrictive as the last. */
                bitmap_copy(bitmap, bitmap_prev, bitmap_size);
        } else {
                /* Before any filters, all syscalls are always allowed. */
                bitmap_fill(bitmap, bitmap_size);
        }

        for (nr = 0; nr < bitmap_size; nr++) {
                /* No bitmap change: not a cacheable action. */
                if (!test_bit(nr, bitmap))
                        continue;

                sd.nr = nr;
                sd.arch = arch;

                /* No bitmap change: continue to always allow. */
                if (seccomp_is_const_allow(fprog, &sd))
                        continue;

                /*
                 * Not a cacheable action: always run filters.
                 * atomic clear_bit() not needed, filter not visible yet.
                 */
                __clear_bit(nr, bitmap);
        }
}

/**
 * seccomp_cache_prepare - emulate the filter to find cacheable syscalls
 * @sfilter: The seccomp filter
 *
 * Returns 0 if successful or -errno if error occurred.
 */
static void seccomp_cache_prepare(struct seccomp_filter *sfilter)
{
        struct action_cache *cache = &sfilter->cache;
        const struct action_cache *cache_prev =
                sfilter->prev ? &sfilter->prev->cache : NULL;

        seccomp_cache_prepare_bitmap(sfilter, cache->allow_native,
                                     cache_prev ? cache_prev->allow_native : NULL,
                                     SECCOMP_ARCH_NATIVE_NR,
                                     SECCOMP_ARCH_NATIVE);

#ifdef SECCOMP_ARCH_COMPAT
        seccomp_cache_prepare_bitmap(sfilter, cache->allow_compat,
                                     cache_prev ? cache_prev->allow_compat : NULL,
                                     SECCOMP_ARCH_COMPAT_NR,
                                     SECCOMP_ARCH_COMPAT);
#endif /* SECCOMP_ARCH_COMPAT */
}
#endif /* SECCOMP_ARCH_NATIVE */

/**
 * seccomp_attach_filter: validate and attach filter
 * @flags:  flags to change filter behavior
 * @filter: seccomp filter to add to the current process
 *
 * Caller must be holding current->sighand->siglock lock.
 *
 * Returns 0 on success, -ve on error, or
 *   - in TSYNC mode: the pid of a thread which was either not in the correct
 *     seccomp mode or did not have an ancestral seccomp filter
 *   - in NEW_LISTENER mode: the fd of the new listener
 */
static long seccomp_attach_filter(unsigned int flags,
                                  struct seccomp_filter *filter)
{
        unsigned long total_insns;
        struct seccomp_filter *walker;

        assert_spin_locked(&current->sighand->siglock);

        /* Validate resulting filter length. */
        total_insns = filter->prog->len;
        for (walker = current->seccomp.filter; walker; walker = walker->prev)
                total_insns += walker->prog->len + 4;  /* 4 instr penalty */
        if (total_insns > MAX_INSNS_PER_PATH)
                return -ENOMEM;

        /* If thread sync has been requested, check that it is possible. */
        if (flags & SECCOMP_FILTER_FLAG_TSYNC) {
                int ret;

                ret = seccomp_can_sync_threads();
                if (ret) {
                        if (flags & SECCOMP_FILTER_FLAG_TSYNC_ESRCH)
                                return -ESRCH;
                        else
                                return ret;
                }
        }

        /* Set log flag, if present. */
        if (flags & SECCOMP_FILTER_FLAG_LOG)
                filter->log = true;

        /* Set wait killable flag, if present. */
        if (flags & SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV)
                filter->wait_killable_recv = true;

        /*
         * If there is an existing filter, make it the prev and don't drop its
         * task reference.
         */
        filter->prev = current->seccomp.filter;
        seccomp_cache_prepare(filter);
        current->seccomp.filter = filter;
        atomic_inc(&current->seccomp.filter_count);

        /* Now that the new filter is in place, synchronize to all threads. */
        if (flags & SECCOMP_FILTER_FLAG_TSYNC)
                seccomp_sync_threads(flags);

        return 0;
}

static void __get_seccomp_filter(struct seccomp_filter *filter)
{
        refcount_inc(&filter->refs);
}

/* get_seccomp_filter - increments the reference count of the filter on @tsk */
void get_seccomp_filter(struct task_struct *tsk)
{
        struct seccomp_filter *orig = tsk->seccomp.filter;
        if (!orig)
                return;
        __get_seccomp_filter(orig);
        refcount_inc(&orig->users);
}

#endif        /* CONFIG_SECCOMP_FILTER */

/* For use with seccomp_actions_logged */
#define SECCOMP_LOG_KILL_PROCESS        (1 << 0)
#define SECCOMP_LOG_KILL_THREAD                (1 << 1)
#define SECCOMP_LOG_TRAP                (1 << 2)
#define SECCOMP_LOG_ERRNO                (1 << 3)
#define SECCOMP_LOG_TRACE                (1 << 4)
#define SECCOMP_LOG_LOG                        (1 << 5)
#define SECCOMP_LOG_ALLOW                (1 << 6)
#define SECCOMP_LOG_USER_NOTIF                (1 << 7)

static u32 seccomp_actions_logged = SECCOMP_LOG_KILL_PROCESS |
                                    SECCOMP_LOG_KILL_THREAD  |
                                    SECCOMP_LOG_TRAP  |
                                    SECCOMP_LOG_ERRNO |
                                    SECCOMP_LOG_USER_NOTIF |
                                    SECCOMP_LOG_TRACE |
                                    SECCOMP_LOG_LOG;

static inline void seccomp_log(unsigned long syscall, long signr, u32 action,
                               bool requested)
{
        bool log = false;

        switch (action) {
        case SECCOMP_RET_ALLOW:
                break;
        case SECCOMP_RET_TRAP:
                log = requested && seccomp_actions_logged & SECCOMP_LOG_TRAP;
                break;
        case SECCOMP_RET_ERRNO:
                log = requested && seccomp_actions_logged & SECCOMP_LOG_ERRNO;
                break;
        case SECCOMP_RET_TRACE:
                log = requested && seccomp_actions_logged & SECCOMP_LOG_TRACE;
                break;
        case SECCOMP_RET_USER_NOTIF:
                log = requested && seccomp_actions_logged & SECCOMP_LOG_USER_NOTIF;
                break;
        case SECCOMP_RET_LOG:
                log = seccomp_actions_logged & SECCOMP_LOG_LOG;
                break;
        case SECCOMP_RET_KILL_THREAD:
                log = seccomp_actions_logged & SECCOMP_LOG_KILL_THREAD;
                break;
        case SECCOMP_RET_KILL_PROCESS:
        default:
                log = seccomp_actions_logged & SECCOMP_LOG_KILL_PROCESS;
        }

        /*
         * Emit an audit message when the action is RET_KILL_*, RET_LOG, or the
         * FILTER_FLAG_LOG bit was set. The admin has the ability to silence
         * any action from being logged by removing the action name from the
         * seccomp_actions_logged sysctl.
         */
        if (!log)
                return;

        audit_seccomp(syscall, signr, action);
}

/*
 * Secure computing mode 1 allows only read/write/exit/sigreturn.
 * To be fully secure this must be combined with rlimit
 * to limit the stack allocations too.
 */
static const int mode1_syscalls[] = {
        __NR_seccomp_read, __NR_seccomp_write, __NR_seccomp_exit, __NR_seccomp_sigreturn,
#ifdef __NR_uretprobe
        __NR_uretprobe,
#endif
#ifdef __NR_uprobe
        __NR_uprobe,
#endif
        -1, /* negative terminated */
};

static void __secure_computing_strict(int this_syscall)
{
        const int *allowed_syscalls = mode1_syscalls;
#ifdef CONFIG_COMPAT
        if (in_compat_syscall())
                allowed_syscalls = get_compat_mode1_syscalls();
#endif
        do {
                if (*allowed_syscalls == this_syscall)
                        return;
        } while (*++allowed_syscalls != -1);

#ifdef SECCOMP_DEBUG
        dump_stack();
#endif
        current->seccomp.mode = SECCOMP_MODE_DEAD;
        seccomp_log(this_syscall, SIGKILL, SECCOMP_RET_KILL_THREAD, true);
        do_exit(SIGKILL);
}

#ifndef CONFIG_HAVE_ARCH_SECCOMP_FILTER
void secure_computing_strict(int this_syscall)
{
        int mode = current->seccomp.mode;

        if (IS_ENABLED(CONFIG_CHECKPOINT_RESTORE) &&
            unlikely(current->ptrace & PT_SUSPEND_SECCOMP))
                return;

        if (mode == SECCOMP_MODE_DISABLED)
                return;
        else if (mode == SECCOMP_MODE_STRICT)
                __secure_computing_strict(this_syscall);
        else
                BUG();
}
int __secure_computing(void)
{
        int this_syscall = syscall_get_nr(current, current_pt_regs());

        secure_computing_strict(this_syscall);
        return 0;
}
#else

#ifdef CONFIG_SECCOMP_FILTER
static u64 seccomp_next_notify_id(struct seccomp_filter *filter)
{
        /*
         * Note: overflow is ok here, the id just needs to be unique per
         * filter.
         */
        lockdep_assert_held(&filter->notify_lock);
        return filter->notif->next_id++;
}

static void seccomp_handle_addfd(struct seccomp_kaddfd *addfd, struct seccomp_knotif *n)
{
        int fd;

        /*
         * Remove the notification, and reset the list pointers, indicating
         * that it has been handled.
         */
        list_del_init(&addfd->list);
        if (!addfd->setfd)
                fd = receive_fd(addfd->file, NULL, addfd->flags);
        else
                fd = receive_fd_replace(addfd->fd, addfd->file, addfd->flags);
        addfd->ret = fd;

        if (addfd->ioctl_flags & SECCOMP_ADDFD_FLAG_SEND) {
                /* If we fail reset and return an error to the notifier */
                if (fd < 0) {
                        n->state = SECCOMP_NOTIFY_SENT;
                } else {
                        /* Return the FD we just added */
                        n->flags = 0;
                        n->error = 0;
                        n->val = fd;
                }
        }

        /*
         * Mark the notification as completed. From this point, addfd mem
         * might be invalidated and we can't safely read it anymore.
         */
        complete(&addfd->completion);
}

static bool should_sleep_killable(struct seccomp_filter *match,
                                  struct seccomp_knotif *n)
{
        return match->wait_killable_recv && n->state >= SECCOMP_NOTIFY_SENT;
}

static int seccomp_do_user_notification(int this_syscall,
                                        struct seccomp_filter *match,
                                        const struct seccomp_data *sd)
{
        int err;
        u32 flags = 0;
        long ret = 0;
        struct seccomp_knotif n = {};
        struct seccomp_kaddfd *addfd, *tmp;

        mutex_lock(&match->notify_lock);
        err = -ENOSYS;
        if (!match->notif)
                goto out;

        n.task = current;
        n.state = SECCOMP_NOTIFY_INIT;
        n.data = sd;
        n.id = seccomp_next_notify_id(match);
        init_completion(&n.ready);
        list_add_tail(&n.list, &match->notif->notifications);
        INIT_LIST_HEAD(&n.addfd);

        atomic_inc(&match->notif->requests);
        if (match->notif->flags & SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP)
                wake_up_poll_on_current_cpu(&match->wqh, EPOLLIN | EPOLLRDNORM);
        else
                wake_up_poll(&match->wqh, EPOLLIN | EPOLLRDNORM);

        /*
         * This is where we wait for a reply from userspace.
         */
        do {
                bool wait_killable = should_sleep_killable(match, &n);

                mutex_unlock(&match->notify_lock);
                if (wait_killable)
                        err = wait_for_completion_killable(&n.ready);
                else
                        err = wait_for_completion_interruptible(&n.ready);
                mutex_lock(&match->notify_lock);

                if (err != 0) {
                        /*
                         * Check to see whether we should switch to wait
                         * killable. Only return the interrupted error if not.
                         */
                        if (!(!wait_killable && should_sleep_killable(match, &n)))
                                goto interrupted;
                }

                addfd = list_first_entry_or_null(&n.addfd,
                                                 struct seccomp_kaddfd, list);
                /* Check if we were woken up by a addfd message */
                if (addfd)
                        seccomp_handle_addfd(addfd, &n);

        }  while (n.state != SECCOMP_NOTIFY_REPLIED);

        ret = n.val;
        err = n.error;
        flags = n.flags;

interrupted:
        /* If there were any pending addfd calls, clear them out */
        list_for_each_entry_safe(addfd, tmp, &n.addfd, list) {
                /* The process went away before we got a chance to handle it */
                addfd->ret = -ESRCH;
                list_del_init(&addfd->list);
                complete(&addfd->completion);
        }

        /*
         * Note that it's possible the listener died in between the time when
         * we were notified of a response (or a signal) and when we were able to
         * re-acquire the lock, so only delete from the list if the
         * notification actually exists.
         *
         * Also note that this test is only valid because there's no way to
         * *reattach* to a notifier right now. If one is added, we'll need to
         * keep track of the notif itself and make sure they match here.
         */
        if (match->notif)
                list_del(&n.list);
out:
        mutex_unlock(&match->notify_lock);

        /* Userspace requests to continue the syscall. */
        if (flags & SECCOMP_USER_NOTIF_FLAG_CONTINUE)
                return 0;

        syscall_set_return_value(current, current_pt_regs(),
                                 err, ret);
        return -1;
}

static int __seccomp_filter(int this_syscall, const bool recheck_after_trace)
{
        u32 filter_ret, action;
        struct seccomp_data sd;
        struct seccomp_filter *match = NULL;
        int data;

        /*
         * Make sure that any changes to mode from another thread have
         * been seen after SYSCALL_WORK_SECCOMP was seen.
         */
        smp_rmb();

        populate_seccomp_data(&sd);

        filter_ret = seccomp_run_filters(&sd, &match);
        data = filter_ret & SECCOMP_RET_DATA;
        action = filter_ret & SECCOMP_RET_ACTION_FULL;

        switch (action) {
        case SECCOMP_RET_ERRNO:
                /* Set low-order bits as an errno, capped at MAX_ERRNO. */
                if (data > MAX_ERRNO)
                        data = MAX_ERRNO;
                syscall_set_return_value(current, current_pt_regs(),
                                         -data, 0);
                goto skip;

        case SECCOMP_RET_TRAP:
                /* Show the handler the original registers. */
                syscall_rollback(current, current_pt_regs());
                /* Let the filter pass back 16 bits of data. */
                force_sig_seccomp(this_syscall, data, false);
                goto skip;

        case SECCOMP_RET_TRACE:
                /* We've been put in this state by the ptracer already. */
                if (recheck_after_trace)
                        return 0;

                /* ENOSYS these calls if there is no tracer attached. */
                if (!ptrace_event_enabled(current, PTRACE_EVENT_SECCOMP)) {
                        syscall_set_return_value(current,
                                                 current_pt_regs(),
                                                 -ENOSYS, 0);
                        goto skip;
                }

                /* Allow the BPF to provide the event message */
                ptrace_event(PTRACE_EVENT_SECCOMP, data);
                /*
                 * The delivery of a fatal signal during event
                 * notification may silently skip tracer notification,
                 * which could leave us with a potentially unmodified
                 * syscall that the tracer would have liked to have
                 * changed. Since the process is about to die, we just
                 * force the syscall to be skipped and let the signal
                 * kill the process and correctly handle any tracer exit
                 * notifications.
                 */
                if (fatal_signal_pending(current))
                        goto skip;
                /* Check if the tracer forced the syscall to be skipped. */
                this_syscall = syscall_get_nr(current, current_pt_regs());
                if (this_syscall < 0)
                        goto skip;

                /*
                 * Recheck the syscall, since it may have changed. This
                 * intentionally uses a NULL struct seccomp_data to force
                 * a reload of all registers. This does not goto skip since
                 * a skip would have already been reported.
                 */
                if (__seccomp_filter(this_syscall, true))
                        return -1;

                return 0;

        case SECCOMP_RET_USER_NOTIF:
                if (seccomp_do_user_notification(this_syscall, match, &sd))
                        goto skip;

                return 0;

        case SECCOMP_RET_LOG:
                seccomp_log(this_syscall, 0, action, true);
                return 0;

        case SECCOMP_RET_ALLOW:
                /*
                 * Note that the "match" filter will always be NULL for
                 * this action since SECCOMP_RET_ALLOW is the starting
                 * state in seccomp_run_filters().
                 */
                return 0;

        case SECCOMP_RET_KILL_THREAD:
        case SECCOMP_RET_KILL_PROCESS:
        default:
                current->seccomp.mode = SECCOMP_MODE_DEAD;
                seccomp_log(this_syscall, SIGSYS, action, true);
                /* Dump core only if this is the last remaining thread. */
                if (action != SECCOMP_RET_KILL_THREAD ||
                    (atomic_read(&current->signal->live) == 1)) {
                        /* Show the original registers in the dump. */
                        syscall_rollback(current, current_pt_regs());
                        /* Trigger a coredump with SIGSYS */
                        force_sig_seccomp(this_syscall, data, true);
                } else {
                        do_exit(SIGSYS);
                }
                return -1; /* skip the syscall go directly to signal handling */
        }

        unreachable();

skip:
        seccomp_log(this_syscall, 0, action, match ? match->log : false);
        return -1;
}
#else
static int __seccomp_filter(int this_syscall, const bool recheck_after_trace)
{
        BUG();

        return -1;
}
#endif

int __secure_computing(void)
{
        int mode = current->seccomp.mode;
        int this_syscall;

        if (IS_ENABLED(CONFIG_CHECKPOINT_RESTORE) &&
            unlikely(current->ptrace & PT_SUSPEND_SECCOMP))
                return 0;

        this_syscall = syscall_get_nr(current, current_pt_regs());

        switch (mode) {
        case SECCOMP_MODE_STRICT:
                __secure_computing_strict(this_syscall);  /* may call do_exit */
                return 0;
        case SECCOMP_MODE_FILTER:
                return __seccomp_filter(this_syscall, false);
        /* Surviving SECCOMP_RET_KILL_* must be proactively impossible. */
        case SECCOMP_MODE_DEAD:
                WARN_ON_ONCE(1);
                do_exit(SIGKILL);
                return -1;
        default:
                BUG();
        }
}
#endif /* CONFIG_HAVE_ARCH_SECCOMP_FILTER */

long prctl_get_seccomp(void)
{
        return current->seccomp.mode;
}

/**
 * seccomp_set_mode_strict: internal function for setting strict seccomp
 *
 * Once current->seccomp.mode is non-zero, it may not be changed.
 *
 * Returns 0 on success or -EINVAL on failure.
 */
static long seccomp_set_mode_strict(void)
{
        const unsigned long seccomp_mode = SECCOMP_MODE_STRICT;
        long ret = -EINVAL;

        spin_lock_irq(&current->sighand->siglock);

        if (!seccomp_may_assign_mode(seccomp_mode))
                goto out;

#ifdef TIF_NOTSC
        disable_TSC();
#endif
        seccomp_assign_mode(current, seccomp_mode, 0);
        ret = 0;

out:
        spin_unlock_irq(&current->sighand->siglock);

        return ret;
}

#ifdef CONFIG_SECCOMP_FILTER
static void seccomp_notify_free(struct seccomp_filter *filter)
{
        kfree(filter->notif);
        filter->notif = NULL;
}

static void seccomp_notify_detach(struct seccomp_filter *filter)
{
        struct seccomp_knotif *knotif;

        if (!filter)
                return;

        mutex_lock(&filter->notify_lock);

        /*
         * If this file is being closed because e.g. the task who owned it
         * died, let's wake everyone up who was waiting on us.
         */
        list_for_each_entry(knotif, &filter->notif->notifications, list) {
                if (knotif->state == SECCOMP_NOTIFY_REPLIED)
                        continue;

                knotif->state = SECCOMP_NOTIFY_REPLIED;
                knotif->error = -ENOSYS;
                knotif->val = 0;

                /*
                 * We do not need to wake up any pending addfd messages, as
                 * the notifier will do that for us, as this just looks
                 * like a standard reply.
                 */
                complete(&knotif->ready);
        }

        seccomp_notify_free(filter);
        mutex_unlock(&filter->notify_lock);
}

static int seccomp_notify_release(struct inode *inode, struct file *file)
{
        struct seccomp_filter *filter = file->private_data;

        seccomp_notify_detach(filter);
        __put_seccomp_filter(filter);
        return 0;
}

/* must be called with notif_lock held */
static inline struct seccomp_knotif *
find_notification(struct seccomp_filter *filter, u64 id)
{
        struct seccomp_knotif *cur;

        lockdep_assert_held(&filter->notify_lock);

        list_for_each_entry(cur, &filter->notif->notifications, list) {
                if (cur->id == id)
                        return cur;
        }

        return NULL;
}

static int recv_wake_function(wait_queue_entry_t *wait, unsigned int mode, int sync,
                                  void *key)
{
        /* Avoid a wakeup if event not interesting for us. */
        if (key && !(key_to_poll(key) & (EPOLLIN | EPOLLERR | EPOLLHUP)))
                return 0;
        return autoremove_wake_function(wait, mode, sync, key);
}

static int recv_wait_event(struct seccomp_filter *filter)
{
        DEFINE_WAIT_FUNC(wait, recv_wake_function);
        int ret;

        if (refcount_read(&filter->users) == 0)
                return 0;

        if (atomic_dec_if_positive(&filter->notif->requests) >= 0)
                return 0;

        for (;;) {
                ret = prepare_to_wait_event(&filter->wqh, &wait, TASK_INTERRUPTIBLE);

                if (atomic_dec_if_positive(&filter->notif->requests) >= 0)
                        break;
                if (refcount_read(&filter->users) == 0)
                        break;

                if (ret)
                        return ret;

                schedule();
        }
        finish_wait(&filter->wqh, &wait);
        return 0;
}

static long seccomp_notify_recv(struct seccomp_filter *filter,
                                void __user *buf)
{
        struct seccomp_knotif *knotif = NULL, *cur;
        struct seccomp_notif unotif;
        ssize_t ret;

        /* Verify that we're not given garbage to keep struct extensible. */
        ret = check_zeroed_user(buf, sizeof(unotif));
        if (ret < 0)
                return ret;
        if (!ret)
                return -EINVAL;

        memset(&unotif, 0, sizeof(unotif));

        ret = recv_wait_event(filter);
        if (ret < 0)
                return ret;

        mutex_lock(&filter->notify_lock);
        list_for_each_entry(cur, &filter->notif->notifications, list) {
                if (cur->state == SECCOMP_NOTIFY_INIT) {
                        knotif = cur;
                        break;
                }
        }

        /*
         * If we didn't find a notification, it could be that the task was
         * interrupted by a fatal signal between the time we were woken and
         * when we were able to acquire the rw lock.
         */
        if (!knotif) {
                ret = -ENOENT;
                goto out;
        }

        unotif.id = knotif->id;
        unotif.pid = task_pid_vnr(knotif->task);
        unotif.data = *(knotif->data);

        knotif->state = SECCOMP_NOTIFY_SENT;
        wake_up_poll(&filter->wqh, EPOLLOUT | EPOLLWRNORM);
        ret = 0;
out:
        mutex_unlock(&filter->notify_lock);

        if (ret == 0 && copy_to_user(buf, &unotif, sizeof(unotif))) {
                ret = -EFAULT;

                /*
                 * Userspace screwed up. To make sure that we keep this
                 * notification alive, let's reset it back to INIT. It
                 * may have died when we released the lock, so we need to make
                 * sure it's still around.
                 */
                mutex_lock(&filter->notify_lock);
                knotif = find_notification(filter, unotif.id);
                if (knotif) {
                        /* Reset the process to make sure it's not stuck */
                        if (should_sleep_killable(filter, knotif))
                                complete(&knotif->ready);
                        knotif->state = SECCOMP_NOTIFY_INIT;
                        atomic_inc(&filter->notif->requests);
                        wake_up_poll(&filter->wqh, EPOLLIN | EPOLLRDNORM);
                }
                mutex_unlock(&filter->notify_lock);
        }

        return ret;
}

static long seccomp_notify_send(struct seccomp_filter *filter,
                                void __user *buf)
{
        struct seccomp_notif_resp resp = {};
        struct seccomp_knotif *knotif;
        long ret;

        if (copy_from_user(&resp, buf, sizeof(resp)))
                return -EFAULT;

        if (resp.flags & ~SECCOMP_USER_NOTIF_FLAG_CONTINUE)
                return -EINVAL;

        if ((resp.flags & SECCOMP_USER_NOTIF_FLAG_CONTINUE) &&
            (resp.error || resp.val))
                return -EINVAL;

        ret = mutex_lock_interruptible(&filter->notify_lock);
        if (ret < 0)
                return ret;

        knotif = find_notification(filter, resp.id);
        if (!knotif) {
                ret = -ENOENT;
                goto out;
        }

        /* Allow exactly one reply. */
        if (knotif->state != SECCOMP_NOTIFY_SENT) {
                ret = -EINPROGRESS;
                goto out;
        }

        ret = 0;
        knotif->state = SECCOMP_NOTIFY_REPLIED;
        knotif->error = resp.error;
        knotif->val = resp.val;
        knotif->flags = resp.flags;
        if (filter->notif->flags & SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP)
                complete_on_current_cpu(&knotif->ready);
        else
                complete(&knotif->ready);
out:
        mutex_unlock(&filter->notify_lock);
        return ret;
}

static long seccomp_notify_id_valid(struct seccomp_filter *filter,
                                    void __user *buf)
{
        struct seccomp_knotif *knotif;
        u64 id;
        long ret;

        if (copy_from_user(&id, buf, sizeof(id)))
                return -EFAULT;

        ret = mutex_lock_interruptible(&filter->notify_lock);
        if (ret < 0)
                return ret;

        knotif = find_notification(filter, id);
        if (knotif && knotif->state == SECCOMP_NOTIFY_SENT)
                ret = 0;
        else
                ret = -ENOENT;

        mutex_unlock(&filter->notify_lock);
        return ret;
}

static long seccomp_notify_set_flags(struct seccomp_filter *filter,
                                    unsigned long flags)
{
        long ret;

        if (flags & ~SECCOMP_USER_NOTIF_FD_SYNC_WAKE_UP)
                return -EINVAL;

        ret = mutex_lock_interruptible(&filter->notify_lock);
        if (ret < 0)
                return ret;
        filter->notif->flags = flags;
        mutex_unlock(&filter->notify_lock);
        return 0;
}

static long seccomp_notify_addfd(struct seccomp_filter *filter,
                                 struct seccomp_notif_addfd __user *uaddfd,
                                 unsigned int size)
{
        struct seccomp_notif_addfd addfd;
        struct seccomp_knotif *knotif;
        struct seccomp_kaddfd kaddfd;
        int ret;

        BUILD_BUG_ON(sizeof(addfd) < SECCOMP_NOTIFY_ADDFD_SIZE_VER0);
        BUILD_BUG_ON(sizeof(addfd) != SECCOMP_NOTIFY_ADDFD_SIZE_LATEST);

        if (size < SECCOMP_NOTIFY_ADDFD_SIZE_VER0 || size >= PAGE_SIZE)
                return -EINVAL;

        ret = copy_struct_from_user(&addfd, sizeof(addfd), uaddfd, size);
        if (ret)
                return ret;

        if (addfd.newfd_flags & ~O_CLOEXEC)
                return -EINVAL;

        if (addfd.flags & ~(SECCOMP_ADDFD_FLAG_SETFD | SECCOMP_ADDFD_FLAG_SEND))
                return -EINVAL;

        if (addfd.newfd && !(addfd.flags & SECCOMP_ADDFD_FLAG_SETFD))
                return -EINVAL;

        kaddfd.file = fget(addfd.srcfd);
        if (!kaddfd.file)
                return -EBADF;

        kaddfd.ioctl_flags = addfd.flags;
        kaddfd.flags = addfd.newfd_flags;
        kaddfd.setfd = addfd.flags & SECCOMP_ADDFD_FLAG_SETFD;
        kaddfd.fd = addfd.newfd;
        init_completion(&kaddfd.completion);

        ret = mutex_lock_interruptible(&filter->notify_lock);
        if (ret < 0)
                goto out;

        knotif = find_notification(filter, addfd.id);
        if (!knotif) {
                ret = -ENOENT;
                goto out_unlock;
        }

        /*
         * We do not want to allow for FD injection to occur before the
         * notification has been picked up by a userspace handler, or after
         * the notification has been replied to.
         */
        if (knotif->state != SECCOMP_NOTIFY_SENT) {
                ret = -EINPROGRESS;
                goto out_unlock;
        }

        if (addfd.flags & SECCOMP_ADDFD_FLAG_SEND) {
                /*
                 * Disallow queuing an atomic addfd + send reply while there are
                 * some addfd requests still to process.
                 *
                 * There is no clear reason to support it and allows us to keep
                 * the loop on the other side straight-forward.
                 */
                if (!list_empty(&knotif->addfd)) {
                        ret = -EBUSY;
                        goto out_unlock;
                }

                /* Allow exactly only one reply */
                knotif->state = SECCOMP_NOTIFY_REPLIED;
        }

        list_add(&kaddfd.list, &knotif->addfd);
        complete(&knotif->ready);
        mutex_unlock(&filter->notify_lock);

        /* Now we wait for it to be processed or be interrupted */
        ret = wait_for_completion_interruptible(&kaddfd.completion);
        if (ret == 0) {
                /*
                 * We had a successful completion. The other side has already
                 * removed us from the addfd queue, and
                 * wait_for_completion_interruptible has a memory barrier upon
                 * success that lets us read this value directly without
                 * locking.
                 */
                ret = kaddfd.ret;
                goto out;
        }

        mutex_lock(&filter->notify_lock);
        /*
         * Even though we were woken up by a signal and not a successful
         * completion, a completion may have happened in the mean time.
         *
         * We need to check again if the addfd request has been handled,
         * and if not, we will remove it from the queue.
         */
        if (list_empty(&kaddfd.list))
                ret = kaddfd.ret;
        else
                list_del(&kaddfd.list);

out_unlock:
        mutex_unlock(&filter->notify_lock);
out:
        fput(kaddfd.file);

        return ret;
}

static long seccomp_notify_ioctl(struct file *file, unsigned int cmd,
                                 unsigned long arg)
{
        struct seccomp_filter *filter = file->private_data;
        void __user *buf = (void __user *)arg;

        /* Fixed-size ioctls */
        switch (cmd) {
        case SECCOMP_IOCTL_NOTIF_RECV:
                return seccomp_notify_recv(filter, buf);
        case SECCOMP_IOCTL_NOTIF_SEND:
                return seccomp_notify_send(filter, buf);
        case SECCOMP_IOCTL_NOTIF_ID_VALID_WRONG_DIR:
        case SECCOMP_IOCTL_NOTIF_ID_VALID:
                return seccomp_notify_id_valid(filter, buf);
        case SECCOMP_IOCTL_NOTIF_SET_FLAGS:
                return seccomp_notify_set_flags(filter, arg);
        }

        /* Extensible Argument ioctls */
#define EA_IOCTL(cmd)        ((cmd) & ~(IOC_INOUT | IOCSIZE_MASK))
        switch (EA_IOCTL(cmd)) {
        case EA_IOCTL(SECCOMP_IOCTL_NOTIF_ADDFD):
                return seccomp_notify_addfd(filter, buf, _IOC_SIZE(cmd));
        default:
                return -EINVAL;
        }
}

static __poll_t seccomp_notify_poll(struct file *file,
                                    struct poll_table_struct *poll_tab)
{
        struct seccomp_filter *filter = file->private_data;
        __poll_t ret = 0;
        struct seccomp_knotif *cur;

        poll_wait(file, &filter->wqh, poll_tab);

        if (mutex_lock_interruptible(&filter->notify_lock) < 0)
                return EPOLLERR;

        list_for_each_entry(cur, &filter->notif->notifications, list) {
                if (cur->state == SECCOMP_NOTIFY_INIT)
                        ret |= EPOLLIN | EPOLLRDNORM;
                if (cur->state == SECCOMP_NOTIFY_SENT)
                        ret |= EPOLLOUT | EPOLLWRNORM;
                if ((ret & EPOLLIN) && (ret & EPOLLOUT))
                        break;
        }

        mutex_unlock(&filter->notify_lock);

        if (refcount_read(&filter->users) == 0)
                ret |= EPOLLHUP;

        return ret;
}

static const struct file_operations seccomp_notify_ops = {
        .poll = seccomp_notify_poll,
        .release = seccomp_notify_release,
        .unlocked_ioctl = seccomp_notify_ioctl,
        .compat_ioctl = seccomp_notify_ioctl,
};

static struct file *init_listener(struct seccomp_filter *filter)
{
        struct file *ret;

        ret = ERR_PTR(-ENOMEM);
        filter->notif = kzalloc(sizeof(*(filter->notif)), GFP_KERNEL);
        if (!filter->notif)
                goto out;

        filter->notif->next_id = get_random_u64();
        INIT_LIST_HEAD(&filter->notif->notifications);

        ret = anon_inode_getfile("seccomp notify", &seccomp_notify_ops,
                                 filter, O_RDWR);
        if (IS_ERR(ret))
                goto out_notif;

        /* The file has a reference to it now */
        __get_seccomp_filter(filter);

out_notif:
        if (IS_ERR(ret))
                seccomp_notify_free(filter);
out:
        return ret;
}

/*
 * Does @new_child have a listener while an ancestor also has a listener?
 * If so, we'll want to reject this filter.
 * This only has to be tested for the current process, even in the TSYNC case,
 * because TSYNC installs @child with the same parent on all threads.
 * Note that @new_child is not hooked up to its parent at this point yet, so
 * we use current->seccomp.filter.
 */
static bool has_duplicate_listener(struct seccomp_filter *new_child)
{
        struct seccomp_filter *cur;

        /* must be protected against concurrent TSYNC */
        lockdep_assert_held(&current->sighand->siglock);

        if (!new_child->notif)
                return false;
        for (cur = current->seccomp.filter; cur; cur = cur->prev) {
                if (cur->notif)
                        return true;
        }

        return false;
}

/**
 * seccomp_set_mode_filter: internal function for setting seccomp filter
 * @flags:  flags to change filter behavior
 * @filter: struct sock_fprog containing filter
 *
 * This function may be called repeatedly to install additional filters.
 * Every filter successfully installed will be evaluated (in reverse order)
 * for each system call the task makes.
 *
 * Once current->seccomp.mode is non-zero, it may not be changed.
 *
 * Returns 0 on success or -EINVAL on failure.
 */
static long seccomp_set_mode_filter(unsigned int flags,
                                    const char __user *filter)
{
        const unsigned long seccomp_mode = SECCOMP_MODE_FILTER;
        struct seccomp_filter *prepared = NULL;
        long ret = -EINVAL;
        int listener = -1;
        struct file *listener_f = NULL;

        /* Validate flags. */
        if (flags & ~SECCOMP_FILTER_FLAG_MASK)
                return -EINVAL;

        /*
         * In the successful case, NEW_LISTENER returns the new listener fd.
         * But in the failure case, TSYNC returns the thread that died. If you
         * combine these two flags, there's no way to tell whether something
         * succeeded or failed. So, let's disallow this combination if the user
         * has not explicitly requested no errors from TSYNC.
         */
        if ((flags & SECCOMP_FILTER_FLAG_TSYNC) &&
            (flags & SECCOMP_FILTER_FLAG_NEW_LISTENER) &&
            ((flags & SECCOMP_FILTER_FLAG_TSYNC_ESRCH) == 0))
                return -EINVAL;

        /*
         * The SECCOMP_FILTER_FLAG_WAIT_KILLABLE_SENT flag doesn't make sense
         * without the SECCOMP_FILTER_FLAG_NEW_LISTENER flag.
         */
        if ((flags & SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV) &&
            ((flags & SECCOMP_FILTER_FLAG_NEW_LISTENER) == 0))
                return -EINVAL;

        /* Prepare the new filter before holding any locks. */
        prepared = seccomp_prepare_user_filter(filter);
        if (IS_ERR(prepared))
                return PTR_ERR(prepared);

        if (flags & SECCOMP_FILTER_FLAG_NEW_LISTENER) {
                listener = get_unused_fd_flags(O_CLOEXEC);
                if (listener < 0) {
                        ret = listener;
                        goto out_free;
                }

                listener_f = init_listener(prepared);
                if (IS_ERR(listener_f)) {
                        put_unused_fd(listener);
                        ret = PTR_ERR(listener_f);
                        goto out_free;
                }
        }

        /*
         * Make sure we cannot change seccomp or nnp state via TSYNC
         * while another thread is in the middle of calling exec.
         */
        if (flags & SECCOMP_FILTER_FLAG_TSYNC &&
            mutex_lock_killable(&current->signal->cred_guard_mutex))
                goto out_put_fd;

        spin_lock_irq(&current->sighand->siglock);

        if (!seccomp_may_assign_mode(seccomp_mode))
                goto out;

        if (has_duplicate_listener(prepared)) {
                ret = -EBUSY;
                goto out;
        }

        ret = seccomp_attach_filter(flags, prepared);
        if (ret)
                goto out;
        /* Do not free the successfully attached filter. */
        prepared = NULL;

        seccomp_assign_mode(current, seccomp_mode, flags);
out:
        spin_unlock_irq(&current->sighand->siglock);
        if (flags & SECCOMP_FILTER_FLAG_TSYNC)
                mutex_unlock(&current->signal->cred_guard_mutex);
out_put_fd:
        if (flags & SECCOMP_FILTER_FLAG_NEW_LISTENER) {
                if (ret) {
                        listener_f->private_data = NULL;
                        fput(listener_f);
                        put_unused_fd(listener);
                        seccomp_notify_detach(prepared);
                } else {
                        fd_install(listener, listener_f);
                        ret = listener;
                }
        }
out_free:
        seccomp_filter_free(prepared);
        return ret;
}
#else
static inline long seccomp_set_mode_filter(unsigned int flags,
                                           const char __user *filter)
{
        return -EINVAL;
}
#endif

static long seccomp_get_action_avail(const char __user *uaction)
{
        u32 action;

        if (copy_from_user(&action, uaction, sizeof(action)))
                return -EFAULT;

        switch (action) {
        case SECCOMP_RET_KILL_PROCESS:
        case SECCOMP_RET_KILL_THREAD:
        case SECCOMP_RET_TRAP:
        case SECCOMP_RET_ERRNO:
        case SECCOMP_RET_USER_NOTIF:
        case SECCOMP_RET_TRACE:
        case SECCOMP_RET_LOG:
        case SECCOMP_RET_ALLOW:
                break;
        default:
                return -EOPNOTSUPP;
        }

        return 0;
}

static long seccomp_get_notif_sizes(void __user *usizes)
{
        struct seccomp_notif_sizes sizes = {
                .seccomp_notif = sizeof(struct seccomp_notif),
                .seccomp_notif_resp = sizeof(struct seccomp_notif_resp),
                .seccomp_data = sizeof(struct seccomp_data),
        };

        if (copy_to_user(usizes, &sizes, sizeof(sizes)))
                return -EFAULT;

        return 0;
}

/* Common entry point for both prctl and syscall. */
static long do_seccomp(unsigned int op, unsigned int flags,
                       void __user *uargs)
{
        switch (op) {
        case SECCOMP_SET_MODE_STRICT:
                if (flags != 0 || uargs != NULL)
                        return -EINVAL;
                return seccomp_set_mode_strict();
        case SECCOMP_SET_MODE_FILTER:
                return seccomp_set_mode_filter(flags, uargs);
        case SECCOMP_GET_ACTION_AVAIL:
                if (flags != 0)
                        return -EINVAL;

                return seccomp_get_action_avail(uargs);
        case SECCOMP_GET_NOTIF_SIZES:
                if (flags != 0)
                        return -EINVAL;

                return seccomp_get_notif_sizes(uargs);
        default:
                return -EINVAL;
        }
}

SYSCALL_DEFINE3(seccomp, unsigned int, op, unsigned int, flags,
                         void __user *, uargs)
{
        return do_seccomp(op, flags, uargs);
}

/**
 * prctl_set_seccomp: configures current->seccomp.mode
 * @seccomp_mode: requested mode to use
 * @filter: optional struct sock_fprog for use with SECCOMP_MODE_FILTER
 *
 * Returns 0 on success or -EINVAL on failure.
 */
long prctl_set_seccomp(unsigned long seccomp_mode, void __user *filter)
{
        unsigned int op;
        void __user *uargs;

        switch (seccomp_mode) {
        case SECCOMP_MODE_STRICT:
                op = SECCOMP_SET_MODE_STRICT;
                /*
                 * Setting strict mode through prctl always ignored filter,
                 * so make sure it is always NULL here to pass the internal
                 * check in do_seccomp().
                 */
                uargs = NULL;
                break;
        case SECCOMP_MODE_FILTER:
                op = SECCOMP_SET_MODE_FILTER;
                uargs = filter;
                break;
        default:
                return -EINVAL;
        }

        /* prctl interface doesn't have flags, so they are always zero. */
        return do_seccomp(op, 0, uargs);
}

#if defined(CONFIG_SECCOMP_FILTER) && defined(CONFIG_CHECKPOINT_RESTORE)
static struct seccomp_filter *get_nth_filter(struct task_struct *task,
                                             unsigned long filter_off)
{
        struct seccomp_filter *orig, *filter;
        unsigned long count;

        /*
         * Note: this is only correct because the caller should be the (ptrace)
         * tracer of the task, otherwise lock_task_sighand is needed.
         */
        spin_lock_irq(&task->sighand->siglock);

        if (task->seccomp.mode != SECCOMP_MODE_FILTER) {
                spin_unlock_irq(&task->sighand->siglock);
                return ERR_PTR(-EINVAL);
        }

        orig = task->seccomp.filter;
        __get_seccomp_filter(orig);
        spin_unlock_irq(&task->sighand->siglock);

        count = 0;
        for (filter = orig; filter; filter = filter->prev)
                count++;

        if (filter_off >= count) {
                filter = ERR_PTR(-ENOENT);
                goto out;
        }

        count -= filter_off;
        for (filter = orig; filter && count > 1; filter = filter->prev)
                count--;

        if (WARN_ON(count != 1 || !filter)) {
                filter = ERR_PTR(-ENOENT);
                goto out;
        }

        __get_seccomp_filter(filter);

out:
        __put_seccomp_filter(orig);
        return filter;
}

long seccomp_get_filter(struct task_struct *task, unsigned long filter_off,
                        void __user *data)
{
        struct seccomp_filter *filter;
        struct sock_fprog_kern *fprog;
        long ret;

        if (!capable(CAP_SYS_ADMIN) ||
            current->seccomp.mode != SECCOMP_MODE_DISABLED) {
                return -EACCES;
        }

        filter = get_nth_filter(task, filter_off);
        if (IS_ERR(filter))
                return PTR_ERR(filter);

        fprog = filter->prog->orig_prog;
        if (!fprog) {
                /* This must be a new non-cBPF filter, since we save
                 * every cBPF filter's orig_prog above when
                 * CONFIG_CHECKPOINT_RESTORE is enabled.
                 */
                ret = -EMEDIUMTYPE;
                goto out;
        }

        ret = fprog->len;
        if (!data)
                goto out;

        if (copy_to_user(data, fprog->filter, bpf_classic_proglen(fprog)))
                ret = -EFAULT;

out:
        __put_seccomp_filter(filter);
        return ret;
}

long seccomp_get_metadata(struct task_struct *task,
                          unsigned long size, void __user *data)
{
        long ret;
        struct seccomp_filter *filter;
        struct seccomp_metadata kmd = {};

        if (!capable(CAP_SYS_ADMIN) ||
            current->seccomp.mode != SECCOMP_MODE_DISABLED) {
                return -EACCES;
        }

        size = min_t(unsigned long, size, sizeof(kmd));

        if (size < sizeof(kmd.filter_off))
                return -EINVAL;

        if (copy_from_user(&kmd.filter_off, data, sizeof(kmd.filter_off)))
                return -EFAULT;

        filter = get_nth_filter(task, kmd.filter_off);
        if (IS_ERR(filter))
                return PTR_ERR(filter);

        if (filter->log)
                kmd.flags |= SECCOMP_FILTER_FLAG_LOG;

        ret = size;
        if (copy_to_user(data, &kmd, size))
                ret = -EFAULT;

        __put_seccomp_filter(filter);
        return ret;
}
#endif

#ifdef CONFIG_SYSCTL

/* Human readable action names for friendly sysctl interaction */
#define SECCOMP_RET_KILL_PROCESS_NAME        "kill_process"
#define SECCOMP_RET_KILL_THREAD_NAME        "kill_thread"
#define SECCOMP_RET_TRAP_NAME                "trap"
#define SECCOMP_RET_ERRNO_NAME                "errno"
#define SECCOMP_RET_USER_NOTIF_NAME        "user_notif"
#define SECCOMP_RET_TRACE_NAME                "trace"
#define SECCOMP_RET_LOG_NAME                "log"
#define SECCOMP_RET_ALLOW_NAME                "allow"

static const char seccomp_actions_avail[] =
                                SECCOMP_RET_KILL_PROCESS_NAME        " "
                                SECCOMP_RET_KILL_THREAD_NAME        " "
                                SECCOMP_RET_TRAP_NAME                " "
                                SECCOMP_RET_ERRNO_NAME                " "
                                SECCOMP_RET_USER_NOTIF_NAME     " "
                                SECCOMP_RET_TRACE_NAME                " "
                                SECCOMP_RET_LOG_NAME                " "
                                SECCOMP_RET_ALLOW_NAME;

struct seccomp_log_name {
        u32                log;
        const char        *name;
};

static const struct seccomp_log_name seccomp_log_names[] = {
        { SECCOMP_LOG_KILL_PROCESS, SECCOMP_RET_KILL_PROCESS_NAME },
        { SECCOMP_LOG_KILL_THREAD, SECCOMP_RET_KILL_THREAD_NAME },
        { SECCOMP_LOG_TRAP, SECCOMP_RET_TRAP_NAME },
        { SECCOMP_LOG_ERRNO, SECCOMP_RET_ERRNO_NAME },
        { SECCOMP_LOG_USER_NOTIF, SECCOMP_RET_USER_NOTIF_NAME },
        { SECCOMP_LOG_TRACE, SECCOMP_RET_TRACE_NAME },
        { SECCOMP_LOG_LOG, SECCOMP_RET_LOG_NAME },
        { SECCOMP_LOG_ALLOW, SECCOMP_RET_ALLOW_NAME },
        { }
};

static bool seccomp_names_from_actions_logged(char *names, size_t size,
                                              u32 actions_logged,
                                              const char *sep)
{
        const struct seccomp_log_name *cur;
        bool append_sep = false;

        for (cur = seccomp_log_names; cur->name && size; cur++) {
                ssize_t ret;

                if (!(actions_logged & cur->log))
                        continue;

                if (append_sep) {
                        ret = strscpy(names, sep, size);
                        if (ret < 0)
                                return false;

                        names += ret;
                        size -= ret;
                } else
                        append_sep = true;

                ret = strscpy(names, cur->name, size);
                if (ret < 0)
                        return false;

                names += ret;
                size -= ret;
        }

        return true;
}

static bool seccomp_action_logged_from_name(u32 *action_logged,
                                            const char *name)
{
        const struct seccomp_log_name *cur;

        for (cur = seccomp_log_names; cur->name; cur++) {
                if (!strcmp(cur->name, name)) {
                        *action_logged = cur->log;
                        return true;
                }
        }

        return false;
}

static bool seccomp_actions_logged_from_names(u32 *actions_logged, char *names)
{
        char *name;

        *actions_logged = 0;
        while ((name = strsep(&names, " ")) && *name) {
                u32 action_logged = 0;

                if (!seccomp_action_logged_from_name(&action_logged, name))
                        return false;

                *actions_logged |= action_logged;
        }

        return true;
}

static int read_actions_logged(const struct ctl_table *ro_table, void *buffer,
                               size_t *lenp, loff_t *ppos)
{
        char names[sizeof(seccomp_actions_avail)];
        struct ctl_table table;

        memset(names, 0, sizeof(names));

        if (!seccomp_names_from_actions_logged(names, sizeof(names),
                                               seccomp_actions_logged, " "))
                return -EINVAL;

        table = *ro_table;
        table.data = names;
        table.maxlen = sizeof(names);
        return proc_dostring(&table, 0, buffer, lenp, ppos);
}

static int write_actions_logged(const struct ctl_table *ro_table, void *buffer,
                                size_t *lenp, loff_t *ppos, u32 *actions_logged)
{
        char names[sizeof(seccomp_actions_avail)];
        struct ctl_table table;
        int ret;

        if (!capable(CAP_SYS_ADMIN))
                return -EPERM;

        memset(names, 0, sizeof(names));

        table = *ro_table;
        table.data = names;
        table.maxlen = sizeof(names);
        ret = proc_dostring(&table, 1, buffer, lenp, ppos);
        if (ret)
                return ret;

        if (!seccomp_actions_logged_from_names(actions_logged, table.data))
                return -EINVAL;

        if (*actions_logged & SECCOMP_LOG_ALLOW)
                return -EINVAL;

        seccomp_actions_logged = *actions_logged;
        return 0;
}

static void audit_actions_logged(u32 actions_logged, u32 old_actions_logged,
                                 int ret)
{
        char names[sizeof(seccomp_actions_avail)];
        char old_names[sizeof(seccomp_actions_avail)];
        const char *new = names;
        const char *old = old_names;

        if (!audit_enabled)
                return;

        memset(names, 0, sizeof(names));
        memset(old_names, 0, sizeof(old_names));

        if (ret)
                new = "?";
        else if (!actions_logged)
                new = "(none)";
        else if (!seccomp_names_from_actions_logged(names, sizeof(names),
                                                    actions_logged, ","))
                new = "?";

        if (!old_actions_logged)
                old = "(none)";
        else if (!seccomp_names_from_actions_logged(old_names,
                                                    sizeof(old_names),
                                                    old_actions_logged, ","))
                old = "?";

        return audit_seccomp_actions_logged(new, old, !ret);
}

static int seccomp_actions_logged_handler(const struct ctl_table *ro_table, int write,
                                          void *buffer, size_t *lenp,
                                          loff_t *ppos)
{
        int ret;

        if (write) {
                u32 actions_logged = 0;
                u32 old_actions_logged = seccomp_actions_logged;

                ret = write_actions_logged(ro_table, buffer, lenp, ppos,
                                           &actions_logged);
                audit_actions_logged(actions_logged, old_actions_logged, ret);
        } else
                ret = read_actions_logged(ro_table, buffer, lenp, ppos);

        return ret;
}

static const struct ctl_table seccomp_sysctl_table[] = {
        {
                .procname        = "actions_avail",
                .data                = (void *) &seccomp_actions_avail,
                .maxlen                = sizeof(seccomp_actions_avail),
                .mode                = 0444,
                .proc_handler        = proc_dostring,
        },
        {
                .procname        = "actions_logged",
                .mode                = 0644,
                .proc_handler        = seccomp_actions_logged_handler,
        },
};

static int __init seccomp_sysctl_init(void)
{
        register_sysctl_init("kernel/seccomp", seccomp_sysctl_table);
        return 0;
}

device_initcall(seccomp_sysctl_init)

#endif /* CONFIG_SYSCTL */

#ifdef CONFIG_SECCOMP_CACHE_DEBUG
/* Currently CONFIG_SECCOMP_CACHE_DEBUG implies SECCOMP_ARCH_NATIVE */
static void proc_pid_seccomp_cache_arch(struct seq_file *m, const char *name,
                                        const void *bitmap, size_t bitmap_size)
{
        int nr;

        for (nr = 0; nr < bitmap_size; nr++) {
                bool cached = test_bit(nr, bitmap);
                char *status = cached ? "ALLOW" : "FILTER";

                seq_printf(m, "%s %d %s\n", name, nr, status);
        }
}

int proc_pid_seccomp_cache(struct seq_file *m, struct pid_namespace *ns,
                           struct pid *pid, struct task_struct *task)
{
        struct seccomp_filter *f;
        unsigned long flags;

        /*
         * We don't want some sandboxed process to know what their seccomp
         * filters consist of.
         */
        if (!file_ns_capable(m->file, &init_user_ns, CAP_SYS_ADMIN))
                return -EACCES;

        if (!lock_task_sighand(task, &flags))
                return -ESRCH;

        f = READ_ONCE(task->seccomp.filter);
        if (!f) {
                unlock_task_sighand(task, &flags);
                return 0;
        }

        /* prevent filter from being freed while we are printing it */
        __get_seccomp_filter(f);
        unlock_task_sighand(task, &flags);

        proc_pid_seccomp_cache_arch(m, SECCOMP_ARCH_NATIVE_NAME,
                                    f->cache.allow_native,
                                    SECCOMP_ARCH_NATIVE_NR);

#ifdef SECCOMP_ARCH_COMPAT
        proc_pid_seccomp_cache_arch(m, SECCOMP_ARCH_COMPAT_NAME,
                                    f->cache.allow_compat,
                                    SECCOMP_ARCH_COMPAT_NR);
#endif /* SECCOMP_ARCH_COMPAT */

        __put_seccomp_filter(f);
        return 0;
}
#endif /* CONFIG_SECCOMP_CACHE_DEBUG */































  116 





    9 
  118 
    6 
  114 



    7 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
// SPDX-License-Identifier: GPL-2.0
/* Multipath TCP cryptographic functions
 * Copyright (c) 2017 - 2019, Intel Corporation.
 *
 * Note: This code is based on mptcp_ctrl.c, mptcp_ipv4.c, and
 *       mptcp_ipv6 from multipath-tcp.org, authored by:
 *
 *       Sébastien Barré <sebastien.barre@uclouvain.be>
 *       Christoph Paasch <christoph.paasch@uclouvain.be>
 *       Jaakko Korkeaniemi <jaakko.korkeaniemi@aalto.fi>
 *       Gregory Detal <gregory.detal@uclouvain.be>
 *       Fabien Duchêne <fabien.duchene@uclouvain.be>
 *       Andreas Seelinger <Andreas.Seelinger@rwth-aachen.de>
 *       Lavkesh Lahngir <lavkesh51@gmail.com>
 *       Andreas Ripke <ripke@neclab.eu>
 *       Vlad Dogaru <vlad.dogaru@intel.com>
 *       Octavian Purdila <octavian.purdila@intel.com>
 *       John Ronan <jronan@tssg.org>
 *       Catalin Nicutar <catalin.nicutar@gmail.com>
 *       Brandon Heller <brandonh@stanford.edu>
 */

#include <linux/kernel.h>
#include <crypto/sha2.h>

#include "protocol.h"

#define SHA256_DIGEST_WORDS (SHA256_DIGEST_SIZE / 4)

void mptcp_crypto_key_sha(u64 key, u32 *token, u64 *idsn)
{
        __be32 mptcp_hashed_key[SHA256_DIGEST_WORDS];
        __be64 input = cpu_to_be64(key);

        sha256((__force u8 *)&input, sizeof(input), (u8 *)mptcp_hashed_key);

        if (token)
                *token = be32_to_cpu(mptcp_hashed_key[0]);
        if (idsn)
                *idsn = be64_to_cpu(*((__be64 *)&mptcp_hashed_key[6]));
}

void mptcp_crypto_hmac_sha(u64 key1, u64 key2, u8 *msg, int len, void *hmac)
{
        __be64 key[2] = { cpu_to_be64(key1), cpu_to_be64(key2) };

        hmac_sha256_usingrawkey((const u8 *)key, sizeof(key), msg, len, hmac);
}

#if IS_MODULE(CONFIG_MPTCP_KUNIT_TEST)
EXPORT_SYMBOL_GPL(mptcp_crypto_hmac_sha);
#endif













































































































































   37 
    1 
  606 











































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __IPC_NAMESPACE_H__
#define __IPC_NAMESPACE_H__

#include <linux/err.h>
#include <linux/idr.h>
#include <linux/rwsem.h>
#include <linux/notifier.h>
#include <linux/nsproxy.h>
#include <linux/ns_common.h>
#include <linux/refcount.h>
#include <linux/rhashtable-types.h>
#include <linux/sysctl.h>
#include <linux/percpu_counter.h>

struct user_namespace;

struct ipc_ids {
        int in_use;
        unsigned short seq;
        struct rw_semaphore rwsem;
        struct idr ipcs_idr;
        int max_idx;
        int last_idx;        /* For wrap around detection */
#ifdef CONFIG_CHECKPOINT_RESTORE
        int next_id;
#endif
        struct rhashtable key_ht;
};

struct ipc_namespace {
        struct ipc_ids        ids[3];

        int                sem_ctls[4];
        int                used_sems;

        unsigned int        msg_ctlmax;
        unsigned int        msg_ctlmnb;
        unsigned int        msg_ctlmni;
        struct percpu_counter percpu_msg_bytes;
        struct percpu_counter percpu_msg_hdrs;

        size_t                shm_ctlmax;
        size_t                shm_ctlall;
        unsigned long        shm_tot;
        int                shm_ctlmni;
        /*
         * Defines whether IPC_RMID is forced for _all_ shm segments regardless
         * of shmctl()
         */
        int                shm_rmid_forced;

        struct notifier_block ipcns_nb;

        /* The kern_mount of the mqueuefs sb.  We take a ref on it */
        struct vfsmount        *mq_mnt;

        /* # queues in this ns, protected by mq_lock */
        unsigned int    mq_queues_count;

        /* next fields are set through sysctl */
        unsigned int    mq_queues_max;   /* initialized to DFLT_QUEUESMAX */
        unsigned int    mq_msg_max;      /* initialized to DFLT_MSGMAX */
        unsigned int    mq_msgsize_max;  /* initialized to DFLT_MSGSIZEMAX */
        unsigned int    mq_msg_default;
        unsigned int    mq_msgsize_default;

        struct ctl_table_set        mq_set;
        struct ctl_table_header        *mq_sysctls;

        struct ctl_table_set        ipc_set;
        struct ctl_table_header        *ipc_sysctls;

        /* user_ns which owns the ipc ns */
        struct user_namespace *user_ns;
        struct ucounts *ucounts;

        struct llist_node mnt_llist;

        struct ns_common ns;
} __randomize_layout;

extern struct ipc_namespace init_ipc_ns;
extern spinlock_t mq_lock;

#ifdef CONFIG_SYSVIPC
extern void shm_destroy_orphaned(struct ipc_namespace *ns);
#else /* CONFIG_SYSVIPC */
static inline void shm_destroy_orphaned(struct ipc_namespace *ns) {}
#endif /* CONFIG_SYSVIPC */

#ifdef CONFIG_POSIX_MQUEUE
extern int mq_init_ns(struct ipc_namespace *ns);
/*
 * POSIX Message Queue default values:
 *
 * MIN_*: Lowest value an admin can set the maximum unprivileged limit to
 * DFLT_*MAX: Default values for the maximum unprivileged limits
 * DFLT_{MSG,MSGSIZE}: Default values used when the user doesn't supply
 *   an attribute to the open call and the queue must be created
 * HARD_*: Highest value the maximums can be set to.  These are enforced
 *   on CAP_SYS_RESOURCE apps as well making them inviolate (so make them
 *   suitably high)
 *
 * POSIX Requirements:
 *   Per app minimum openable message queues - 8.  This does not map well
 *     to the fact that we limit the number of queues on a per namespace
 *     basis instead of a per app basis.  So, make the default high enough
 *     that no given app should have a hard time opening 8 queues.
 *   Minimum maximum for HARD_MSGMAX - 32767.  I bumped this to 65536.
 *   Minimum maximum for HARD_MSGSIZEMAX - POSIX is silent on this.  However,
 *     we have run into a situation where running applications in the wild
 *     require this to be at least 5MB, and preferably 10MB, so I set the
 *     value to 16MB in hopes that this user is the worst of the bunch and
 *     the new maximum will handle anyone else.  I may have to revisit this
 *     in the future.
 */
#define DFLT_QUEUESMAX                      256
#define MIN_MSGMAX                        1
#define DFLT_MSG                       10U
#define DFLT_MSGMAX                       10
#define HARD_MSGMAX                    65536
#define MIN_MSGSIZEMAX                      128
#define DFLT_MSGSIZE                     8192U
#define DFLT_MSGSIZEMAX                     8192
#define HARD_MSGSIZEMAX            (16*1024*1024)
#else
static inline int mq_init_ns(struct ipc_namespace *ns) { return 0; }
#endif

#if defined(CONFIG_IPC_NS)
static inline struct ipc_namespace *to_ipc_ns(struct ns_common *ns)
{
        return container_of(ns, struct ipc_namespace, ns);
}

extern struct ipc_namespace *copy_ipcs(u64 flags,
        struct user_namespace *user_ns, struct ipc_namespace *ns);

static inline struct ipc_namespace *get_ipc_ns(struct ipc_namespace *ns)
{
        if (ns)
                ns_ref_inc(ns);
        return ns;
}

static inline struct ipc_namespace *get_ipc_ns_not_zero(struct ipc_namespace *ns)
{
        if (ns) {
                if (ns_ref_get(ns))
                        return ns;
        }

        return NULL;
}

extern void put_ipc_ns(struct ipc_namespace *ns);
#else
static inline struct ipc_namespace *copy_ipcs(u64 flags,
        struct user_namespace *user_ns, struct ipc_namespace *ns)
{
        if (flags & CLONE_NEWIPC)
                return ERR_PTR(-EINVAL);

        return ns;
}

static inline struct ipc_namespace *get_ipc_ns(struct ipc_namespace *ns)
{
        return ns;
}

static inline struct ipc_namespace *get_ipc_ns_not_zero(struct ipc_namespace *ns)
{
        return ns;
}

static inline void put_ipc_ns(struct ipc_namespace *ns)
{
}
#endif

#ifdef CONFIG_POSIX_MQUEUE_SYSCTL

void retire_mq_sysctls(struct ipc_namespace *ns);
bool setup_mq_sysctls(struct ipc_namespace *ns);

#else /* CONFIG_POSIX_MQUEUE_SYSCTL */

static inline void retire_mq_sysctls(struct ipc_namespace *ns)
{
}

static inline bool setup_mq_sysctls(struct ipc_namespace *ns)
{
        return true;
}

#endif /* CONFIG_POSIX_MQUEUE_SYSCTL */

#ifdef CONFIG_SYSVIPC_SYSCTL

bool setup_ipc_sysctls(struct ipc_namespace *ns);
void retire_ipc_sysctls(struct ipc_namespace *ns);

#else /* CONFIG_SYSVIPC_SYSCTL */

static inline void retire_ipc_sysctls(struct ipc_namespace *ns)
{
}

static inline bool setup_ipc_sysctls(struct ipc_namespace *ns)
{
        return true;
}

#endif /* CONFIG_SYSVIPC_SYSCTL */
#endif



















  152 




   65 

    1 


   87 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
// SPDX-License-Identifier: GPL-2.0-or-later
/* 32-bit compatibility syscall for 64-bit systems for DH operations
 *
 * Copyright (C) 2016 Stephan Mueller <smueller@chronox.de>
 */

#include <linux/uaccess.h>

#include "internal.h"

/*
 * Perform the DH computation or DH based key derivation.
 *
 * If successful, 0 will be returned.
 */
long compat_keyctl_dh_compute(struct keyctl_dh_params __user *params,
                              char __user *buffer, size_t buflen,
                              struct compat_keyctl_kdf_params __user *kdf)
{
        struct keyctl_kdf_params kdfcopy;
        struct compat_keyctl_kdf_params compat_kdfcopy;

        if (!kdf)
                return __keyctl_dh_compute(params, buffer, buflen, NULL);

        if (copy_from_user(&compat_kdfcopy, kdf, sizeof(compat_kdfcopy)) != 0)
                return -EFAULT;

        kdfcopy.hashname = compat_ptr(compat_kdfcopy.hashname);
        kdfcopy.otherinfo = compat_ptr(compat_kdfcopy.otherinfo);
        kdfcopy.otherinfolen = compat_kdfcopy.otherinfolen;
        memcpy(kdfcopy.__spare, compat_kdfcopy.__spare,
               sizeof(kdfcopy.__spare));

        return __keyctl_dh_compute(params, buffer, buflen, &kdfcopy);
}

































































   22 
   22 
















































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
// SPDX-License-Identifier: GPL-2.0
/*
 * This file contains the base functions to manage periodic tick
 * related events.
 *
 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
 */
#include <linux/compiler.h>
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/hrtimer.h>
#include <linux/interrupt.h>
#include <linux/nmi.h>
#include <linux/percpu.h>
#include <linux/profile.h>
#include <linux/sched.h>
#include <linux/module.h>
#include <trace/events/power.h>

#include <asm/irq_regs.h>

#include "tick-internal.h"

/*
 * Tick devices
 */
DEFINE_PER_CPU(struct tick_device, tick_cpu_device);
/*
 * Tick next event: keeps track of the tick time. It's updated by the
 * CPU which handles the tick and protected by jiffies_lock. There is
 * no requirement to write hold the jiffies seqcount for it.
 */
ktime_t tick_next_period;

/*
 * tick_do_timer_cpu is a timer core internal variable which holds the CPU NR
 * which is responsible for calling do_timer(), i.e. the timekeeping stuff. This
 * variable has two functions:
 *
 * 1) Prevent a thundering herd issue of a gazillion of CPUs trying to grab the
 *    timekeeping lock all at once. Only the CPU which is assigned to do the
 *    update is handling it.
 *
 * 2) Hand off the duty in the NOHZ idle case by setting the value to
 *    TICK_DO_TIMER_NONE, i.e. a non existing CPU. So the next cpu which looks
 *    at it will take over and keep the time keeping alive.  The handover
 *    procedure also covers cpu hotplug.
 */
int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT;
#ifdef CONFIG_NO_HZ_FULL
/*
 * tick_do_timer_boot_cpu indicates the boot CPU temporarily owns
 * tick_do_timer_cpu and it should be taken over by an eligible secondary
 * when one comes online.
 */
static int tick_do_timer_boot_cpu __read_mostly = -1;
#endif

/*
 * Debugging: see timer_list.c
 */
struct tick_device *tick_get_device(int cpu)
{
        return &per_cpu(tick_cpu_device, cpu);
}

/**
 * tick_is_oneshot_available - check for a oneshot capable event device
 */
int tick_is_oneshot_available(void)
{
        struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);

        if (!dev || !(dev->features & CLOCK_EVT_FEAT_ONESHOT))
                return 0;
        if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
                return 1;
        return tick_broadcast_oneshot_available();
}

/*
 * Periodic tick
 */
static void tick_periodic(int cpu)
{
        if (READ_ONCE(tick_do_timer_cpu) == cpu) {
                raw_spin_lock(&jiffies_lock);
                write_seqcount_begin(&jiffies_seq);

                /* Keep track of the next tick event */
                tick_next_period = ktime_add_ns(tick_next_period, TICK_NSEC);

                do_timer(1);
                write_seqcount_end(&jiffies_seq);
                raw_spin_unlock(&jiffies_lock);
                update_wall_time();
        }

        update_process_times(user_mode(get_irq_regs()));
        profile_tick(CPU_PROFILING);
}

/*
 * Event handler for periodic ticks
 */
void tick_handle_periodic(struct clock_event_device *dev)
{
        int cpu = smp_processor_id();
        ktime_t next = dev->next_event;

        tick_periodic(cpu);

        /*
         * The cpu might have transitioned to HIGHRES or NOHZ mode via
         * update_process_times() -> run_local_timers() ->
         * hrtimer_run_queues().
         */
        if (IS_ENABLED(CONFIG_TICK_ONESHOT) && dev->event_handler != tick_handle_periodic)
                return;

        if (!clockevent_state_oneshot(dev))
                return;
        for (;;) {
                /*
                 * Setup the next period for devices, which do not have
                 * periodic mode:
                 */
                next = ktime_add_ns(next, TICK_NSEC);

                if (!clockevents_program_event(dev, next, false))
                        return;
                /*
                 * Have to be careful here. If we're in oneshot mode,
                 * before we call tick_periodic() in a loop, we need
                 * to be sure we're using a real hardware clocksource.
                 * Otherwise we could get trapped in an infinite
                 * loop, as the tick_periodic() increments jiffies,
                 * which then will increment time, possibly causing
                 * the loop to trigger again and again.
                 */
                if (timekeeping_valid_for_hres())
                        tick_periodic(cpu);
        }
}

/*
 * Setup the device for a periodic tick
 */
void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
{
        tick_set_periodic_handler(dev, broadcast);

        /* Broadcast setup ? */
        if (!tick_device_is_functional(dev))
                return;

        if ((dev->features & CLOCK_EVT_FEAT_PERIODIC) &&
            !tick_broadcast_oneshot_active()) {
                clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);
        } else {
                unsigned int seq;
                ktime_t next;

                do {
                        seq = read_seqcount_begin(&jiffies_seq);
                        next = tick_next_period;
                } while (read_seqcount_retry(&jiffies_seq, seq));

                clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);

                for (;;) {
                        if (!clockevents_program_event(dev, next, false))
                                return;
                        next = ktime_add_ns(next, TICK_NSEC);
                }
        }
}

/*
 * Setup the tick device
 */
static void tick_setup_device(struct tick_device *td,
                              struct clock_event_device *newdev, int cpu,
                              const struct cpumask *cpumask)
{
        void (*handler)(struct clock_event_device *) = NULL;
        ktime_t next_event = 0;

        /*
         * First device setup ?
         */
        if (!td->evtdev) {
                /*
                 * If no cpu took the do_timer update, assign it to
                 * this cpu:
                 */
                if (READ_ONCE(tick_do_timer_cpu) == TICK_DO_TIMER_BOOT) {
                        WRITE_ONCE(tick_do_timer_cpu, cpu);
                        tick_next_period = ktime_get();
#ifdef CONFIG_NO_HZ_FULL
                        /*
                         * The boot CPU may be nohz_full, in which case the
                         * first housekeeping secondary will take do_timer()
                         * from it.
                         */
                        if (tick_nohz_full_cpu(cpu))
                                tick_do_timer_boot_cpu = cpu;

                } else if (tick_do_timer_boot_cpu != -1 && !tick_nohz_full_cpu(cpu)) {
                        tick_do_timer_boot_cpu = -1;
                        /*
                         * The boot CPU will stay in periodic (NOHZ disabled)
                         * mode until clocksource_done_booting() called after
                         * smp_init() selects a high resolution clocksource and
                         * timekeeping_notify() kicks the NOHZ stuff alive.
                         *
                         * So this WRITE_ONCE can only race with the READ_ONCE
                         * check in tick_periodic() but this race is harmless.
                         */
                        WRITE_ONCE(tick_do_timer_cpu, cpu);
#endif
                }

                /*
                 * Startup in periodic mode first.
                 */
                td->mode = TICKDEV_MODE_PERIODIC;
        } else {
                handler = td->evtdev->event_handler;
                next_event = td->evtdev->next_event;
                td->evtdev->event_handler = clockevents_handle_noop;
        }

        td->evtdev = newdev;

        /*
         * When the device is not per cpu, pin the interrupt to the
         * current cpu:
         */
        if (!cpumask_equal(newdev->cpumask, cpumask))
                irq_set_affinity(newdev->irq, cpumask);

        /*
         * When global broadcasting is active, check if the current
         * device is registered as a placeholder for broadcast mode.
         * This allows us to handle this x86 misfeature in a generic
         * way. This function also returns !=0 when we keep the
         * current active broadcast state for this CPU.
         */
        if (tick_device_uses_broadcast(newdev, cpu))
                return;

        if (td->mode == TICKDEV_MODE_PERIODIC)
                tick_setup_periodic(newdev, 0);
        else
                tick_setup_oneshot(newdev, handler, next_event);
}

void tick_install_replacement(struct clock_event_device *newdev)
{
        struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
        int cpu = smp_processor_id();

        clockevents_exchange_device(td->evtdev, newdev);
        tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
        if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
                tick_oneshot_notify();
}

static bool tick_check_percpu(struct clock_event_device *curdev,
                              struct clock_event_device *newdev, int cpu)
{
        if (!cpumask_test_cpu(cpu, newdev->cpumask))
                return false;
        if (cpumask_equal(newdev->cpumask, cpumask_of(cpu)))
                return true;
        /* Check if irq affinity can be set */
        if (newdev->irq >= 0 && !irq_can_set_affinity(newdev->irq))
                return false;
        /* Prefer an existing cpu local device */
        if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu)))
                return false;
        return true;
}

static bool tick_check_preferred(struct clock_event_device *curdev,
                                 struct clock_event_device *newdev)
{
        /* Prefer oneshot capable device */
        if (!(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) {
                if (curdev && (curdev->features & CLOCK_EVT_FEAT_ONESHOT))
                        return false;
                if (tick_oneshot_mode_active())
                        return false;
        }

        /*
         * Use the higher rated one, but prefer a CPU local device with a lower
         * rating than a non-CPU local device
         */
        return !curdev ||
                newdev->rating > curdev->rating ||
               !cpumask_equal(curdev->cpumask, newdev->cpumask);
}

/*
 * Check whether the new device is a better fit than curdev. curdev
 * can be NULL !
 */
bool tick_check_replacement(struct clock_event_device *curdev,
                            struct clock_event_device *newdev)
{
        if (!tick_check_percpu(curdev, newdev, smp_processor_id()))
                return false;

        return tick_check_preferred(curdev, newdev);
}

/*
 * Check, if the new registered device should be used. Called with
 * clockevents_lock held and interrupts disabled.
 */
void tick_check_new_device(struct clock_event_device *newdev)
{
        struct clock_event_device *curdev;
        struct tick_device *td;
        int cpu;

        cpu = smp_processor_id();
        td = &per_cpu(tick_cpu_device, cpu);
        curdev = td->evtdev;

        if (!tick_check_replacement(curdev, newdev))
                goto out_bc;

        if (!try_module_get(newdev->owner))
                return;

        /*
         * Replace the eventually existing device by the new
         * device. If the current device is the broadcast device, do
         * not give it back to the clockevents layer !
         */
        if (tick_is_broadcast_device(curdev)) {
                clockevents_shutdown(curdev);
                curdev = NULL;
        }
        clockevents_exchange_device(curdev, newdev);
        tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
        if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
                tick_oneshot_notify();
        return;

out_bc:
        /*
         * Can the new device be used as a broadcast device ?
         */
        tick_install_broadcast_device(newdev, cpu);
}

/**
 * tick_broadcast_oneshot_control - Enter/exit broadcast oneshot mode
 * @state:        The target state (enter/exit)
 *
 * The system enters/leaves a state, where affected devices might stop
 * Returns 0 on success, -EBUSY if the cpu is used to broadcast wakeups.
 *
 * Called with interrupts disabled, so clockevents_lock is not
 * required here because the local clock event device cannot go away
 * under us.
 */
int tick_broadcast_oneshot_control(enum tick_broadcast_state state)
{
        struct tick_device *td = this_cpu_ptr(&tick_cpu_device);

        if (!(td->evtdev->features & CLOCK_EVT_FEAT_C3STOP))
                return 0;

        return __tick_broadcast_oneshot_control(state);
}
EXPORT_SYMBOL_GPL(tick_broadcast_oneshot_control);

#ifdef CONFIG_HOTPLUG_CPU
void tick_assert_timekeeping_handover(void)
{
        WARN_ON_ONCE(tick_do_timer_cpu == smp_processor_id());
}
/*
 * Stop the tick and transfer the timekeeping job away from a dying cpu.
 */
int tick_cpu_dying(unsigned int dying_cpu)
{
        /*
         * If the current CPU is the timekeeper, it's the only one that can
         * safely hand over its duty. Also all online CPUs are in stop
         * machine, guaranteed not to be idle, therefore there is no
         * concurrency and it's safe to pick any online successor.
         */
        if (tick_do_timer_cpu == dying_cpu)
                tick_do_timer_cpu = cpumask_first(cpu_online_mask);

        /* Make sure the CPU won't try to retake the timekeeping duty */
        tick_sched_timer_dying(dying_cpu);

        /* Remove CPU from timer broadcasting */
        tick_offline_cpu(dying_cpu);

        return 0;
}

/*
 * Shutdown an event device on the outgoing CPU:
 *
 * Called by the dying CPU during teardown, with clockevents_lock held
 * and interrupts disabled.
 */
void tick_shutdown(void)
{
        struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
        struct clock_event_device *dev = td->evtdev;

        td->mode = TICKDEV_MODE_PERIODIC;
        if (dev) {
                clockevents_exchange_device(dev, NULL);
                dev->event_handler = clockevents_handle_noop;
                td->evtdev = NULL;
        }
}
#endif

/**
 * tick_suspend_local - Suspend the local tick device
 *
 * Called from the local cpu for freeze with interrupts disabled.
 *
 * No locks required. Nothing can change the per cpu device.
 */
void tick_suspend_local(void)
{
        struct tick_device *td = this_cpu_ptr(&tick_cpu_device);

        clockevents_shutdown(td->evtdev);
}

/**
 * tick_resume_local - Resume the local tick device
 *
 * Called from the local CPU for unfreeze or XEN resume magic.
 *
 * No locks required. Nothing can change the per cpu device.
 */
void tick_resume_local(void)
{
        struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
        bool broadcast = tick_resume_check_broadcast();

        clockevents_tick_resume(td->evtdev);
        if (!broadcast) {
                if (td->mode == TICKDEV_MODE_PERIODIC)
                        tick_setup_periodic(td->evtdev, 0);
                else
                        tick_resume_oneshot();
        }

        /*
         * Ensure that hrtimers are up to date and the clockevents device
         * is reprogrammed correctly when high resolution timers are
         * enabled.
         */
        hrtimers_resume_local();
}

/**
 * tick_suspend - Suspend the tick and the broadcast device
 *
 * Called from syscore_suspend() via timekeeping_suspend with only one
 * CPU online and interrupts disabled or from tick_unfreeze() under
 * tick_freeze_lock.
 *
 * No locks required. Nothing can change the per cpu device.
 */
void tick_suspend(void)
{
        tick_suspend_local();
        tick_suspend_broadcast();
}

/**
 * tick_resume - Resume the tick and the broadcast device
 *
 * Called from syscore_resume() via timekeeping_resume with only one
 * CPU online and interrupts disabled.
 *
 * No locks required. Nothing can change the per cpu device.
 */
void tick_resume(void)
{
        tick_resume_broadcast();
        tick_resume_local();
}

#ifdef CONFIG_SUSPEND
static DEFINE_RAW_SPINLOCK(tick_freeze_lock);
static DEFINE_WAIT_OVERRIDE_MAP(tick_freeze_map, LD_WAIT_SLEEP);
static unsigned int tick_freeze_depth;

/**
 * tick_freeze - Suspend the local tick and (possibly) timekeeping.
 *
 * Check if this is the last online CPU executing the function and if so,
 * suspend timekeeping.  Otherwise suspend the local tick.
 *
 * Call with interrupts disabled.  Must be balanced with %tick_unfreeze().
 * Interrupts must not be enabled before the subsequent %tick_unfreeze().
 */
void tick_freeze(void)
{
        raw_spin_lock(&tick_freeze_lock);

        tick_freeze_depth++;
        if (tick_freeze_depth == num_online_cpus()) {
                trace_suspend_resume(TPS("timekeeping_freeze"),
                                     smp_processor_id(), true);
                /*
                 * All other CPUs have their interrupts disabled and are
                 * suspended to idle. Other tasks have been frozen so there
                 * is no scheduling happening. This means that there is no
                 * concurrency in the system at this point. Therefore it is
                 * okay to acquire a sleeping lock on PREEMPT_RT, such as a
                 * spinlock, because the lock cannot be held by other CPUs
                 * or threads and acquiring it cannot block.
                 *
                 * Inform lockdep about the situation.
                 */
                lock_map_acquire_try(&tick_freeze_map);
                system_state = SYSTEM_SUSPEND;
                sched_clock_suspend();
                timekeeping_suspend();
                lock_map_release(&tick_freeze_map);
        } else {
                tick_suspend_local();
        }

        raw_spin_unlock(&tick_freeze_lock);
}

/**
 * tick_unfreeze - Resume the local tick and (possibly) timekeeping.
 *
 * Check if this is the first CPU executing the function and if so, resume
 * timekeeping.  Otherwise resume the local tick.
 *
 * Call with interrupts disabled.  Must be balanced with %tick_freeze().
 * Interrupts must not be enabled after the preceding %tick_freeze().
 */
void tick_unfreeze(void)
{
        raw_spin_lock(&tick_freeze_lock);

        if (tick_freeze_depth == num_online_cpus()) {
                /*
                 * Similar to tick_freeze(). On resumption the first CPU may
                 * acquire uncontended sleeping locks while other CPUs block on
                 * tick_freeze_lock.
                 */
                lock_map_acquire_try(&tick_freeze_map);
                timekeeping_resume();
                sched_clock_resume();
                lock_map_release(&tick_freeze_map);

                system_state = SYSTEM_RUNNING;
                trace_suspend_resume(TPS("timekeeping_freeze"),
                                     smp_processor_id(), false);
        } else {
                touch_softlockup_watchdog();
                tick_resume_local();
        }

        tick_freeze_depth--;

        raw_spin_unlock(&tick_freeze_lock);
}
#endif /* CONFIG_SUSPEND */

/**
 * tick_init - initialize the tick control
 */
void __init tick_init(void)
{
        tick_broadcast_init();
        tick_nohz_init();
}



























   22 




























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_PREEMPT_H
#define __ASM_PREEMPT_H

#include <asm/rmwcc.h>
#include <asm/percpu.h>

#include <linux/static_call_types.h>

DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count);

/* We use the MSB mostly because its available */
#define PREEMPT_NEED_RESCHED        0x80000000

/*
 * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
 * that a decrement hitting 0 means we can and should reschedule.
 */
#define PREEMPT_ENABLED        (0 + PREEMPT_NEED_RESCHED)

/*
 * We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users
 * that think a non-zero value indicates we cannot preempt.
 */
static __always_inline int preempt_count(void)
{
        return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
}

static __always_inline void preempt_count_set(int pc)
{
        int old, new;

        old = raw_cpu_read_4(__preempt_count);
        do {
                new = (old & PREEMPT_NEED_RESCHED) |
                        (pc & ~PREEMPT_NEED_RESCHED);
        } while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new));
}

/*
 * must be macros to avoid header recursion hell
 */
#define init_task_preempt_count(p) do { } while (0)

#define init_idle_preempt_count(p, cpu) do { \
        per_cpu(__preempt_count, (cpu)) = PREEMPT_DISABLED; \
} while (0)

/*
 * We fold the NEED_RESCHED bit into the preempt count such that
 * preempt_enable() can decrement and test for needing to reschedule with a
 * single instruction.
 *
 * We invert the actual bit, so that when the decrement hits 0 we know we both
 * need to resched (the bit is cleared) and can resched (no preempt count).
 */

static __always_inline void set_preempt_need_resched(void)
{
        raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
}

static __always_inline void clear_preempt_need_resched(void)
{
        raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
}

static __always_inline bool test_preempt_need_resched(void)
{
        return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
}

/*
 * The various preempt_count add/sub methods
 */

static __always_inline void __preempt_count_add(int val)
{
        raw_cpu_add_4(__preempt_count, val);
}

static __always_inline void __preempt_count_sub(int val)
{
        raw_cpu_add_4(__preempt_count, -val);
}

/*
 * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule
 * a decrement which hits zero means we have no preempt_count and should
 * reschedule.
 */
static __always_inline bool __preempt_count_dec_and_test(void)
{
        return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e,
                               __percpu_arg([var]));
}

/*
 * Returns true when we need to resched and can (barring IRQ state).
 */
static __always_inline bool should_resched(int preempt_offset)
{
        return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
}

#ifdef CONFIG_PREEMPTION

extern asmlinkage void preempt_schedule(void);
extern asmlinkage void preempt_schedule_thunk(void);

#define preempt_schedule_dynamic_enabled        preempt_schedule_thunk
#define preempt_schedule_dynamic_disabled        NULL

extern asmlinkage void preempt_schedule_notrace(void);
extern asmlinkage void preempt_schedule_notrace_thunk(void);

#define preempt_schedule_notrace_dynamic_enabled        preempt_schedule_notrace_thunk
#define preempt_schedule_notrace_dynamic_disabled        NULL

#ifdef CONFIG_PREEMPT_DYNAMIC

DECLARE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);

#define __preempt_schedule() \
do { \
        __STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule); \
        asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule) : ASM_CALL_CONSTRAINT); \
} while (0)

DECLARE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);

#define __preempt_schedule_notrace() \
do { \
        __STATIC_CALL_MOD_ADDRESSABLE(preempt_schedule_notrace); \
        asm volatile ("call " STATIC_CALL_TRAMP_STR(preempt_schedule_notrace) : ASM_CALL_CONSTRAINT); \
} while (0)

#else /* PREEMPT_DYNAMIC */

#define __preempt_schedule() \
        asm volatile ("call preempt_schedule_thunk" : ASM_CALL_CONSTRAINT);

#define __preempt_schedule_notrace() \
        asm volatile ("call preempt_schedule_notrace_thunk" : ASM_CALL_CONSTRAINT);

#endif /* PREEMPT_DYNAMIC */

#endif /* PREEMPTION */

#endif /* __ASM_PREEMPT_H */







































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  199 


  199 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* -
 * net/sched/act_ct.c  Connection Tracking action
 *
 * Authors:   Paul Blakey <paulb@mellanox.com>
 *            Yossi Kuperman <yossiku@mellanox.com>
 *            Marcelo Ricardo Leitner <marcelo.leitner@gmail.com>
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/pkt_cls.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/rhashtable.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/act_api.h>
#include <net/ip.h>
#include <net/ipv6_frag.h>
#include <uapi/linux/tc_act/tc_ct.h>
#include <net/tc_act/tc_ct.h>
#include <net/tc_wrapper.h>

#include <net/netfilter/nf_flow_table.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_core.h>
#include <net/netfilter/nf_conntrack_zones.h>
#include <net/netfilter/nf_conntrack_helper.h>
#include <net/netfilter/nf_conntrack_acct.h>
#include <net/netfilter/ipv6/nf_defrag_ipv6.h>
#include <net/netfilter/nf_co