Total coverage: 105271 (6%)of 1904131 



























































































































































    2 


    2 
































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
// SPDX-License-Identifier: GPL-2.0-only
/*
 *        xt_ipvs - kernel module to match IPVS connection properties
 *
 *        Author: Hannes Eder <heder@google.com>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/spinlock.h>
#include <linux/skbuff.h>
#ifdef CONFIG_IP_VS_IPV6
#include <net/ipv6.h>
#endif
#include <linux/ip_vs.h>
#include <linux/types.h>
#include <linux/netfilter/x_tables.h>
#include <linux/netfilter/xt_ipvs.h>
#include <net/netfilter/nf_conntrack.h>

#include <net/ip_vs.h>

MODULE_AUTHOR("Hannes Eder <heder@google.com>");
MODULE_DESCRIPTION("Xtables: match IPVS connection properties");
MODULE_LICENSE("GPL");
MODULE_ALIAS("ipt_ipvs");
MODULE_ALIAS("ip6t_ipvs");

/* borrowed from xt_conntrack */
static bool ipvs_mt_addrcmp(const union nf_inet_addr *kaddr,
                            const union nf_inet_addr *uaddr,
                            const union nf_inet_addr *umask,
                            unsigned int l3proto)
{
        if (l3proto == NFPROTO_IPV4)
                return ((kaddr->ip ^ uaddr->ip) & umask->ip) == 0;
#ifdef CONFIG_IP_VS_IPV6
        else if (l3proto == NFPROTO_IPV6)
                return ipv6_masked_addr_cmp(&kaddr->in6, &umask->in6,
                       &uaddr->in6) == 0;
#endif
        else
                return false;
}

static bool
ipvs_mt(const struct sk_buff *skb, struct xt_action_param *par)
{
        const struct xt_ipvs_mtinfo *data = par->matchinfo;
        struct netns_ipvs *ipvs = net_ipvs(xt_net(par));
        /* ipvs_mt_check ensures that family is only NFPROTO_IPV[46]. */
        const u_int8_t family = xt_family(par);
        struct ip_vs_iphdr iph;
        struct ip_vs_protocol *pp;
        struct ip_vs_conn *cp;
        bool match = true;

        if (data->bitmask == XT_IPVS_IPVS_PROPERTY) {
                match = skb->ipvs_property ^
                        !!(data->invert & XT_IPVS_IPVS_PROPERTY);
                goto out;
        }

        /* other flags than XT_IPVS_IPVS_PROPERTY are set */
        if (!skb->ipvs_property) {
                match = false;
                goto out;
        }

        ip_vs_fill_iph_skb(family, skb, true, &iph);

        if (data->bitmask & XT_IPVS_PROTO)
                if ((iph.protocol == data->l4proto) ^
                    !(data->invert & XT_IPVS_PROTO)) {
                        match = false;
                        goto out;
                }

        pp = ip_vs_proto_get(iph.protocol);
        if (unlikely(!pp)) {
                match = false;
                goto out;
        }

        /*
         * Check if the packet belongs to an existing entry
         */
        cp = pp->conn_out_get(ipvs, family, skb, &iph);
        if (unlikely(cp == NULL)) {
                match = false;
                goto out;
        }

        /*
         * We found a connection, i.e. ct != 0, make sure to call
         * __ip_vs_conn_put before returning.  In our case jump to out_put_con.
         */

        if (data->bitmask & XT_IPVS_VPORT)
                if ((cp->vport == data->vport) ^
                    !(data->invert & XT_IPVS_VPORT)) {
                        match = false;
                        goto out_put_cp;
                }

        if (data->bitmask & XT_IPVS_VPORTCTL)
                if ((cp->control != NULL &&
                     cp->control->vport == data->vportctl) ^
                    !(data->invert & XT_IPVS_VPORTCTL)) {
                        match = false;
                        goto out_put_cp;
                }

        if (data->bitmask & XT_IPVS_DIR) {
                enum ip_conntrack_info ctinfo;
                struct nf_conn *ct = nf_ct_get(skb, &ctinfo);

                if (ct == NULL) {
                        match = false;
                        goto out_put_cp;
                }

                if ((ctinfo >= IP_CT_IS_REPLY) ^
                    !!(data->invert & XT_IPVS_DIR)) {
                        match = false;
                        goto out_put_cp;
                }
        }

        if (data->bitmask & XT_IPVS_METHOD)
                if (((cp->flags & IP_VS_CONN_F_FWD_MASK) == data->fwd_method) ^
                    !(data->invert & XT_IPVS_METHOD)) {
                        match = false;
                        goto out_put_cp;
                }

        if (data->bitmask & XT_IPVS_VADDR) {
                if (ipvs_mt_addrcmp(&cp->vaddr, &data->vaddr,
                                    &data->vmask, family) ^
                    !(data->invert & XT_IPVS_VADDR)) {
                        match = false;
                        goto out_put_cp;
                }
        }

out_put_cp:
        __ip_vs_conn_put(cp);
out:
        pr_debug("match=%d\n", match);
        return match;
}

static int ipvs_mt_check(const struct xt_mtchk_param *par)
{
        if (par->family != NFPROTO_IPV4
#ifdef CONFIG_IP_VS_IPV6
            && par->family != NFPROTO_IPV6
#endif
                ) {
                pr_info_ratelimited("protocol family %u not supported\n",
                                    par->family);
                return -EINVAL;
        }

        return 0;
}

static struct xt_match xt_ipvs_mt_reg __read_mostly = {
        .name       = "ipvs",
        .revision   = 0,
        .family     = NFPROTO_UNSPEC,
        .match      = ipvs_mt,
        .checkentry = ipvs_mt_check,
        .matchsize  = XT_ALIGN(sizeof(struct xt_ipvs_mtinfo)),
        .me         = THIS_MODULE,
};

static int __init ipvs_mt_init(void)
{
        return xt_register_match(&xt_ipvs_mt_reg);
}

static void __exit ipvs_mt_exit(void)
{
        xt_unregister_match(&xt_ipvs_mt_reg);
}

module_init(ipvs_mt_init);
module_exit(ipvs_mt_exit);































   20 


   20 
   20 
   20 





   13 


   13 
   13 
   13 





   13 


   13 
   13 
   13 





   13 


   13 
   13 
   13 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Copyright (c) 2014 Jiri Pirko <jiri@resnulli.us>
 */

#ifndef __NET_TC_VLAN_H
#define __NET_TC_VLAN_H

#include <net/act_api.h>
#include <linux/tc_act/tc_vlan.h>

struct tcf_vlan_params {
        int                  action;
        int               tcfv_action;
        unsigned char     tcfv_push_dst[ETH_ALEN];
        unsigned char     tcfv_push_src[ETH_ALEN];
        u16               tcfv_push_vid;
        __be16            tcfv_push_proto;
        u8                tcfv_push_prio;
        bool              tcfv_push_prio_exists;
        struct rcu_head   rcu;
};

struct tcf_vlan {
        struct tc_action        common;
        struct tcf_vlan_params __rcu *vlan_p;
};
#define to_vlan(a) ((struct tcf_vlan *)a)

static inline u32 tcf_vlan_action(const struct tc_action *a)
{
        u32 tcfv_action;

        rcu_read_lock();
        tcfv_action = rcu_dereference(to_vlan(a)->vlan_p)->tcfv_action;
        rcu_read_unlock();

        return tcfv_action;
}

static inline u16 tcf_vlan_push_vid(const struct tc_action *a)
{
        u16 tcfv_push_vid;

        rcu_read_lock();
        tcfv_push_vid = rcu_dereference(to_vlan(a)->vlan_p)->tcfv_push_vid;
        rcu_read_unlock();

        return tcfv_push_vid;
}

static inline __be16 tcf_vlan_push_proto(const struct tc_action *a)
{
        __be16 tcfv_push_proto;

        rcu_read_lock();
        tcfv_push_proto = rcu_dereference(to_vlan(a)->vlan_p)->tcfv_push_proto;
        rcu_read_unlock();

        return tcfv_push_proto;
}

static inline u8 tcf_vlan_push_prio(const struct tc_action *a)
{
        u8 tcfv_push_prio;

        rcu_read_lock();
        tcfv_push_prio = rcu_dereference(to_vlan(a)->vlan_p)->tcfv_push_prio;
        rcu_read_unlock();

        return tcfv_push_prio;
}

static inline void tcf_vlan_push_eth(unsigned char *src, unsigned char *dest,
                                     const struct tc_action *a)
{
        rcu_read_lock();
        memcpy(dest, rcu_dereference(to_vlan(a)->vlan_p)->tcfv_push_dst, ETH_ALEN);
        memcpy(src, rcu_dereference(to_vlan(a)->vlan_p)->tcfv_push_src, ETH_ALEN);
        rcu_read_unlock();
}

#endif /* __NET_TC_VLAN_H */




































































   38 




    2 


   34 

    1 

    1 


   33 

    1 

    1 









   14 








   14 










    1 


    1 
















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
// SPDX-License-Identifier: GPL-2.0-only
#include <net/ip.h>
#include <net/tcp.h>

#include <net/netfilter/nf_tables.h>
#include <linux/netfilter/nfnetlink_osf.h>

struct nft_osf {
        u8                        dreg;
        u8                        ttl;
        u32                        flags;
};

static const struct nla_policy nft_osf_policy[NFTA_OSF_MAX + 1] = {
        [NFTA_OSF_DREG]                = NLA_POLICY_MAX(NLA_BE32, NFT_REG32_MAX),
        [NFTA_OSF_TTL]                = { .type = NLA_U8 },
        [NFTA_OSF_FLAGS]        = NLA_POLICY_MASK(NLA_BE32, NFT_OSF_F_VERSION),
};

static void nft_osf_eval(const struct nft_expr *expr, struct nft_regs *regs,
                         const struct nft_pktinfo *pkt)
{
        struct nft_osf *priv = nft_expr_priv(expr);
        u32 *dest = &regs->data[priv->dreg];
        struct sk_buff *skb = pkt->skb;
        char os_match[NFT_OSF_MAXGENRELEN];
        const struct tcphdr *tcp;
        struct nf_osf_data data;
        struct tcphdr _tcph;

        if (nft_pf(pkt) != NFPROTO_IPV4) {
                regs->verdict.code = NFT_BREAK;
                return;
        }

        if (pkt->tprot != IPPROTO_TCP || pkt->fragoff) {
                regs->verdict.code = NFT_BREAK;
                return;
        }

        tcp = skb_header_pointer(skb, ip_hdrlen(skb),
                                 sizeof(struct tcphdr), &_tcph);
        if (!tcp) {
                regs->verdict.code = NFT_BREAK;
                return;
        }
        if (!tcp->syn) {
                regs->verdict.code = NFT_BREAK;
                return;
        }

        if (!nf_osf_find(skb, nf_osf_fingers, priv->ttl, &data)) {
                strscpy_pad((char *)dest, "unknown", NFT_OSF_MAXGENRELEN);
        } else {
                if (priv->flags & NFT_OSF_F_VERSION)
                        snprintf(os_match, NFT_OSF_MAXGENRELEN, "%s:%s",
                                 data.genre, data.version);
                else
                        strscpy(os_match, data.genre, NFT_OSF_MAXGENRELEN);

                strscpy_pad((char *)dest, os_match, NFT_OSF_MAXGENRELEN);
        }
}

static int nft_osf_init(const struct nft_ctx *ctx,
                        const struct nft_expr *expr,
                        const struct nlattr * const tb[])
{
        struct nft_osf *priv = nft_expr_priv(expr);
        u32 flags;
        u8 ttl;

        if (!tb[NFTA_OSF_DREG])
                return -EINVAL;

        if (tb[NFTA_OSF_TTL]) {
                ttl = nla_get_u8(tb[NFTA_OSF_TTL]);
                if (ttl > 2)
                        return -EINVAL;
                priv->ttl = ttl;
        }

        if (tb[NFTA_OSF_FLAGS]) {
                flags = ntohl(nla_get_be32(tb[NFTA_OSF_FLAGS]));
                if (flags != NFT_OSF_F_VERSION)
                        return -EINVAL;
                priv->flags = flags;
        }

        return nft_parse_register_store(ctx, tb[NFTA_OSF_DREG], &priv->dreg,
                                        NULL, NFT_DATA_VALUE,
                                        NFT_OSF_MAXGENRELEN);
}

static int nft_osf_dump(struct sk_buff *skb,
                        const struct nft_expr *expr, bool reset)
{
        const struct nft_osf *priv = nft_expr_priv(expr);

        if (nla_put_u8(skb, NFTA_OSF_TTL, priv->ttl))
                goto nla_put_failure;

        if (nla_put_u32(skb, NFTA_OSF_FLAGS, ntohl((__force __be32)priv->flags)))
                goto nla_put_failure;

        if (nft_dump_register(skb, NFTA_OSF_DREG, priv->dreg))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -1;
}

static int nft_osf_validate(const struct nft_ctx *ctx,
                            const struct nft_expr *expr)
{
        unsigned int hooks;

        switch (ctx->family) {
        case NFPROTO_IPV4:
        case NFPROTO_INET:
                hooks = (1 << NF_INET_LOCAL_IN) |
                        (1 << NF_INET_PRE_ROUTING) |
                        (1 << NF_INET_FORWARD);
                break;
        default:
                return -EOPNOTSUPP;
        }

        return nft_chain_validate_hooks(ctx->chain, hooks);
}

static struct nft_expr_type nft_osf_type;
static const struct nft_expr_ops nft_osf_op = {
        .eval                = nft_osf_eval,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_osf)),
        .init                = nft_osf_init,
        .dump                = nft_osf_dump,
        .type                = &nft_osf_type,
        .validate        = nft_osf_validate,
};

static struct nft_expr_type nft_osf_type __read_mostly = {
        .ops                = &nft_osf_op,
        .name                = "osf",
        .owner                = THIS_MODULE,
        .policy                = nft_osf_policy,
        .maxattr        = NFTA_OSF_MAX,
};

static int __init nft_osf_module_init(void)
{
        return nft_register_expr(&nft_osf_type);
}

static void __exit nft_osf_module_exit(void)
{
        return nft_unregister_expr(&nft_osf_type);
}

module_init(nft_osf_module_init);
module_exit(nft_osf_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Fernando Fernandez <ffmancera@riseup.net>");
MODULE_ALIAS_NFT_EXPR("osf");
MODULE_DESCRIPTION("nftables passive OS fingerprint support");

















    5 



























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCHED_COREDUMP_H
#define _LINUX_SCHED_COREDUMP_H

#include <linux/mm_types.h>

#define SUID_DUMP_DISABLE        0        /* No setuid dumping */
#define SUID_DUMP_USER                1        /* Dump as user of process */
#define SUID_DUMP_ROOT                2        /* Dump as root */

static inline unsigned long __mm_flags_get_dumpable(const struct mm_struct *mm)
{
        /*
         * By convention, dumpable bits are contained in first 32 bits of the
         * bitmap, so we can simply access this first unsigned long directly.
         */
        return __mm_flags_get_word(mm);
}

static inline void __mm_flags_set_mask_dumpable(struct mm_struct *mm, int value)
{
        __mm_flags_set_mask_bits_word(mm, MMF_DUMPABLE_MASK, value);
}

extern void set_dumpable(struct mm_struct *mm, int value);
/*
 * This returns the actual value of the suid_dumpable flag. For things
 * that are using this for checking for privilege transitions, it must
 * test against SUID_DUMP_USER rather than treating it as a boolean
 * value.
 */
static inline int __get_dumpable(unsigned long mm_flags)
{
        return mm_flags & MMF_DUMPABLE_MASK;
}

static inline int get_dumpable(struct mm_struct *mm)
{
        unsigned long flags = __mm_flags_get_dumpable(mm);

        return __get_dumpable(flags);
}

#endif /* _LINUX_SCHED_COREDUMP_H */























































































   19 




   10 


    9 






    9 



    5 



    4 
































    2 



    3 

    1 


    3 




    2 























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
/* vcan.c - Virtual CAN interface
 *
 * Copyright (c) 2002-2017 Volkswagen Group Electronic Research
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of Volkswagen nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * Alternatively, provided that this notice is retained in full, this
 * software may be distributed under the terms of the GNU General
 * Public License ("GPL") version 2, in which case the provisions of the
 * GPL apply INSTEAD OF those given above.
 *
 * The provided data structures and external interfaces from this code
 * are not restricted to be used by modules with a GPL compatible license.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 *
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/ethtool.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/if_ether.h>
#include <linux/can.h>
#include <linux/can/can-ml.h>
#include <linux/can/dev.h>
#include <linux/can/skb.h>
#include <linux/slab.h>
#include <net/rtnetlink.h>

#define DRV_NAME "vcan"

MODULE_DESCRIPTION("virtual CAN interface");
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Urs Thuermann <urs.thuermann@volkswagen.de>");
MODULE_ALIAS_RTNL_LINK(DRV_NAME);

/* CAN test feature:
 * Enable the echo on driver level for testing the CAN core echo modes.
 * See Documentation/networking/can.rst for details.
 */

static bool echo; /* echo testing. Default: 0 (Off) */
module_param(echo, bool, 0444);
MODULE_PARM_DESC(echo, "Echo sent frames (for testing). Default: 0 (Off)");

static void vcan_rx(struct sk_buff *skb, struct net_device *dev)
{
        struct net_device_stats *stats = &dev->stats;

        stats->rx_packets++;
        stats->rx_bytes += can_skb_get_data_len(skb);

        skb->pkt_type  = PACKET_BROADCAST;
        skb->dev       = dev;
        skb->ip_summed = CHECKSUM_UNNECESSARY;

        netif_rx(skb);
}

static netdev_tx_t vcan_tx(struct sk_buff *skb, struct net_device *dev)
{
        struct net_device_stats *stats = &dev->stats;
        unsigned int len;
        int loop;

        if (can_dropped_invalid_skb(dev, skb))
                return NETDEV_TX_OK;

        len = can_skb_get_data_len(skb);
        stats->tx_packets++;
        stats->tx_bytes += len;

        /* set flag whether this packet has to be looped back */
        loop = skb->pkt_type == PACKET_LOOPBACK;

        skb_tx_timestamp(skb);

        if (!echo) {
                /* no echo handling available inside this driver */
                if (loop) {
                        /* only count the packets here, because the
                         * CAN core already did the echo for us
                         */
                        stats->rx_packets++;
                        stats->rx_bytes += len;
                }
                consume_skb(skb);
                return NETDEV_TX_OK;
        }

        /* perform standard echo handling for CAN network interfaces */

        if (loop) {
                skb = can_create_echo_skb(skb);
                if (!skb)
                        return NETDEV_TX_OK;

                /* receive with packet counting */
                vcan_rx(skb, dev);
        } else {
                /* no looped packets => no counting */
                consume_skb(skb);
        }
        return NETDEV_TX_OK;
}

static void vcan_set_cap_info(struct net_device *dev)
{
        u32 can_cap = CAN_CAP_CC;

        if (dev->mtu > CAN_MTU)
                can_cap |= CAN_CAP_FD;

        if (dev->mtu >= CANXL_MIN_MTU)
                can_cap |= CAN_CAP_XL;

        can_set_cap(dev, can_cap);
}

static int vcan_change_mtu(struct net_device *dev, int new_mtu)
{
        /* Do not allow changing the MTU while running */
        if (dev->flags & IFF_UP)
                return -EBUSY;

        if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU &&
            !can_is_canxl_dev_mtu(new_mtu))
                return -EINVAL;

        WRITE_ONCE(dev->mtu, new_mtu);
        vcan_set_cap_info(dev);
        return 0;
}

static const struct net_device_ops vcan_netdev_ops = {
        .ndo_start_xmit = vcan_tx,
        .ndo_change_mtu = vcan_change_mtu,
};

static const struct ethtool_ops vcan_ethtool_ops = {
        .get_ts_info = ethtool_op_get_ts_info,
};

static void vcan_setup(struct net_device *dev)
{
        dev->type                = ARPHRD_CAN;
        dev->mtu                = CANXL_MTU;
        dev->hard_header_len        = 0;
        dev->addr_len                = 0;
        dev->tx_queue_len        = 0;
        dev->flags                = IFF_NOARP;
        can_set_ml_priv(dev, netdev_priv(dev));
        vcan_set_cap_info(dev);

        /* set flags according to driver capabilities */
        if (echo)
                dev->flags |= IFF_ECHO;

        dev->netdev_ops                = &vcan_netdev_ops;
        dev->ethtool_ops        = &vcan_ethtool_ops;
        dev->needs_free_netdev        = true;
}

static struct rtnl_link_ops vcan_link_ops __read_mostly = {
        .kind = DRV_NAME,
        .priv_size = sizeof(struct can_ml_priv),
        .setup = vcan_setup,
};

static __init int vcan_init_module(void)
{
        pr_info("Virtual CAN interface driver\n");

        if (echo)
                pr_info("enabled echo on driver level.\n");

        return rtnl_link_register(&vcan_link_ops);
}

static __exit void vcan_cleanup_module(void)
{
        rtnl_link_unregister(&vcan_link_ops);
}

module_init(vcan_init_module);
module_exit(vcan_cleanup_module);


































































































































































  589 






    1 



    1 





















































  616 
  615 



  616 
  616 








 4198 











 4194 


 4203 









 2456 



















 1688 
























  616 








  616 















  616 







  555 


















  574 
















 3473 
 1688 





































































































  616 

















   13 








   13 








 4198 




 4189 

 4198 



































  604 








  604 












































  193 
















  193 







  193 














































































  193 

  192 




  193 

  193 

  193 
  193 
  193 

















































































































































































  616 




  616 








  616 
































  616 























































































   55 





























  605 





























  605 

















  605 









  605 



































   55 




   55 
   55 



















   96 


   41 
   55 




























































































































  612 


  613 

  613 












   16 








    4 
   12 















   16 







    4 
   12 







   22 









   22 










   22 































   31 





   31 































































































  590 

















  605 

























  590 










  605 









   44 



  565 







  605 















  128 







  479 




  604 

















  604 






























  605 










  604 












































  508 


















  508 







  508 





















































































   75 







   75 























    3 






   72 



















  508 

  507 














































  622 











    1 









    6 
















  616 







  616 





    1 

















   32 


  590 




  616 




























  616 



    3 







  615 

  613 







  616 


  616 


  616 



  614 






  615 

  615 


   13 



  605 















  605 
























  603 




















  604 




  605 






































































   14 

  547 



    2 







  546 










  485 












    1 
















  509 














  508 




  509 

























    2 















  508 


































   19 

  491 



















  508 


















  508 












  440 
   74 














  508 

  507 
  508 

















  507 




  485 
   24 

























    2 





   39 



   41 





   55 








































   16 




































































  622 












  625 













  624 
    1 



    2 








   35 
  590 














   24 





  508 




  442 
   68 

  479 


   32 




  507 







  508 


  479 
   31 
   31 





















































































  624 























































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/kernel/fork.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/*
 *  'fork.c' contains the help-routines for the 'fork' system call
 * (see also entry.S and others).
 * Fork is rather simple, once you get the hang of it, but the memory
 * management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
 */

#include <linux/anon_inodes.h>
#include <linux/slab.h>
#include <linux/sched/autogroup.h>
#include <linux/sched/mm.h>
#include <linux/sched/user.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/stat.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/sched/cputime.h>
#include <linux/sched/ext.h>
#include <linux/seq_file.h>
#include <linux/rtmutex.h>
#include <linux/init.h>
#include <linux/unistd.h>
#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/completion.h>
#include <linux/personality.h>
#include <linux/mempolicy.h>
#include <linux/sem.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/iocontext.h>
#include <linux/key.h>
#include <linux/kmsan.h>
#include <linux/binfmts.h>
#include <linux/mman.h>
#include <linux/mmu_notifier.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/memblock.h>
#include <linux/nsproxy.h>
#include <linux/ns/ns_common_types.h>
#include <linux/capability.h>
#include <linux/cpu.h>
#include <linux/cgroup.h>
#include <linux/security.h>
#include <linux/hugetlb.h>
#include <linux/seccomp.h>
#include <linux/swap.h>
#include <linux/syscalls.h>
#include <linux/syscall_user_dispatch.h>
#include <linux/jiffies.h>
#include <linux/futex.h>
#include <linux/compat.h>
#include <linux/kthread.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/rcupdate.h>
#include <linux/ptrace.h>
#include <linux/mount.h>
#include <linux/audit.h>
#include <linux/memcontrol.h>
#include <linux/ftrace.h>
#include <linux/proc_fs.h>
#include <linux/profile.h>
#include <linux/rmap.h>
#include <linux/ksm.h>
#include <linux/acct.h>
#include <linux/userfaultfd_k.h>
#include <linux/tsacct_kern.h>
#include <linux/cn_proc.h>
#include <linux/freezer.h>
#include <linux/delayacct.h>
#include <linux/taskstats_kern.h>
#include <linux/tty.h>
#include <linux/fs_struct.h>
#include <linux/magic.h>
#include <linux/perf_event.h>
#include <linux/posix-timers.h>
#include <linux/user-return-notifier.h>
#include <linux/oom.h>
#include <linux/khugepaged.h>
#include <linux/signalfd.h>
#include <linux/uprobes.h>
#include <linux/aio.h>
#include <linux/compiler.h>
#include <linux/sysctl.h>
#include <linux/kcov.h>
#include <linux/livepatch.h>
#include <linux/thread_info.h>
#include <linux/kstack_erase.h>
#include <linux/kasan.h>
#include <linux/randomize_kstack.h>
#include <linux/scs.h>
#include <linux/io_uring.h>
#include <linux/io_uring_types.h>
#include <linux/bpf.h>
#include <linux/stackprotector.h>
#include <linux/user_events.h>
#include <linux/iommu.h>
#include <linux/rseq.h>
#include <uapi/linux/pidfd.h>
#include <linux/pidfs.h>
#include <linux/tick.h>
#include <linux/unwind_deferred.h>
#include <linux/pgalloc.h>
#include <linux/uaccess.h>

#include <asm/mmu_context.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>

/* For dup_mmap(). */
#include "../mm/internal.h"

#include <trace/events/sched.h>

#define CREATE_TRACE_POINTS
#include <trace/events/task.h>

#include <kunit/visibility.h>

/*
 * Minimum number of threads to boot the kernel
 */
#define MIN_THREADS 20

/*
 * Maximum number of threads
 */
#define MAX_THREADS FUTEX_TID_MASK

/*
 * Protected counters by write_lock_irq(&tasklist_lock)
 */
unsigned long total_forks;        /* Handle normal Linux uptimes. */
int nr_threads;                        /* The idle threads do not count.. */

static int max_threads;                /* tunable limit on nr_threads */

#define NAMED_ARRAY_INDEX(x)        [x] = __stringify(x)

static const char * const resident_page_types[] = {
        NAMED_ARRAY_INDEX(MM_FILEPAGES),
        NAMED_ARRAY_INDEX(MM_ANONPAGES),
        NAMED_ARRAY_INDEX(MM_SWAPENTS),
        NAMED_ARRAY_INDEX(MM_SHMEMPAGES),
};

DEFINE_PER_CPU(unsigned long, process_counts) = 0;

__cacheline_aligned DEFINE_RWLOCK(tasklist_lock);  /* outer */

#ifdef CONFIG_PROVE_RCU
int lockdep_tasklist_lock_is_held(void)
{
        return lockdep_is_held(&tasklist_lock);
}
EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held);
#endif /* #ifdef CONFIG_PROVE_RCU */

int nr_processes(void)
{
        int cpu;
        int total = 0;

        for_each_possible_cpu(cpu)
                total += per_cpu(process_counts, cpu);

        return total;
}

void __weak arch_release_task_struct(struct task_struct *tsk)
{
}

static struct kmem_cache *task_struct_cachep;

static inline struct task_struct *alloc_task_struct_node(int node)
{
        return kmem_cache_alloc_node(task_struct_cachep, GFP_KERNEL, node);
}

static inline void free_task_struct(struct task_struct *tsk)
{
        kmem_cache_free(task_struct_cachep, tsk);
}

#ifdef CONFIG_VMAP_STACK
/*
 * vmalloc() is a bit slow, and calling vfree() enough times will force a TLB
 * flush.  Try to minimize the number of calls by caching stacks.
 */
#define NR_CACHED_STACKS 2
static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
/*
 * Allocated stacks are cached and later reused by new threads, so memcg
 * accounting is performed by the code assigning/releasing stacks to tasks.
 * We need a zeroed memory without __GFP_ACCOUNT.
 */
#define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO)

struct vm_stack {
        struct rcu_head rcu;
        struct vm_struct *stack_vm_area;
};

static struct vm_struct *alloc_thread_stack_node_from_cache(struct task_struct *tsk, int node)
{
        struct vm_struct *vm_area;
        unsigned int i;

        /*
         * If the node has memory, we are guaranteed the stacks are backed by local pages.
         * Otherwise the pages are arbitrary.
         *
         * Note that depending on cpuset it is possible we will get migrated to a different
         * node immediately after allocating here, so this does *not* guarantee locality for
         * arbitrary callers.
         */
        scoped_guard(preempt) {
                if (node != NUMA_NO_NODE && numa_node_id() != node)
                        return NULL;

                for (i = 0; i < NR_CACHED_STACKS; i++) {
                        vm_area = this_cpu_xchg(cached_stacks[i], NULL);
                        if (vm_area)
                                return vm_area;
                }
        }

        return NULL;
}

static bool try_release_thread_stack_to_cache(struct vm_struct *vm_area)
{
        unsigned int i;
        int nid;

        /*
         * Don't cache stacks if any of the pages don't match the local domain, unless
         * there is no local memory to begin with.
         *
         * Note that lack of local memory does not automatically mean it makes no difference
         * performance-wise which other domain backs the stack. In this case we are merely
         * trying to avoid constantly going to vmalloc.
         */
        scoped_guard(preempt) {
                nid = numa_node_id();
                if (node_state(nid, N_MEMORY)) {
                        for (i = 0; i < vm_area->nr_pages; i++) {
                                struct page *page = vm_area->pages[i];
                                if (page_to_nid(page) != nid)
                                        return false;
                        }
                }

                for (i = 0; i < NR_CACHED_STACKS; i++) {
                        struct vm_struct *tmp = NULL;

                        if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm_area))
                                return true;
                }
        }
        return false;
}

static void thread_stack_free_rcu(struct rcu_head *rh)
{
        struct vm_stack *vm_stack = container_of(rh, struct vm_stack, rcu);
        struct vm_struct *vm_area = vm_stack->stack_vm_area;

        if (try_release_thread_stack_to_cache(vm_stack->stack_vm_area))
                return;

        vfree(vm_area->addr);
}

static void thread_stack_delayed_free(struct task_struct *tsk)
{
        struct vm_stack *vm_stack = tsk->stack;

        vm_stack->stack_vm_area = tsk->stack_vm_area;
        call_rcu(&vm_stack->rcu, thread_stack_free_rcu);
}

static int free_vm_stack_cache(unsigned int cpu)
{
        struct vm_struct **cached_vm_stack_areas = per_cpu_ptr(cached_stacks, cpu);
        int i;

        for (i = 0; i < NR_CACHED_STACKS; i++) {
                struct vm_struct *vm_area = cached_vm_stack_areas[i];

                if (!vm_area)
                        continue;

                vfree(vm_area->addr);
                cached_vm_stack_areas[i] = NULL;
        }

        return 0;
}

static int memcg_charge_kernel_stack(struct vm_struct *vm_area)
{
        int i;
        int ret;
        int nr_charged = 0;

        BUG_ON(vm_area->nr_pages != THREAD_SIZE / PAGE_SIZE);

        for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++) {
                ret = memcg_kmem_charge_page(vm_area->pages[i], GFP_KERNEL, 0);
                if (ret)
                        goto err;
                nr_charged++;
        }
        return 0;
err:
        for (i = 0; i < nr_charged; i++)
                memcg_kmem_uncharge_page(vm_area->pages[i], 0);
        return ret;
}

static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
        struct vm_struct *vm_area;
        void *stack;

        vm_area = alloc_thread_stack_node_from_cache(tsk, node);
        if (vm_area) {
                if (memcg_charge_kernel_stack(vm_area)) {
                        vfree(vm_area->addr);
                        return -ENOMEM;
                }

                /* Reset stack metadata. */
                kasan_unpoison_range(vm_area->addr, THREAD_SIZE);

                stack = kasan_reset_tag(vm_area->addr);

                /* Clear stale pointers from reused stack. */
                clear_pages(vm_area->addr, vm_area->nr_pages);

                tsk->stack_vm_area = vm_area;
                tsk->stack = stack;
                return 0;
        }

        stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN,
                                     GFP_VMAP_STACK,
                                     node, __builtin_return_address(0));
        if (!stack)
                return -ENOMEM;

        vm_area = find_vm_area(stack);
        if (memcg_charge_kernel_stack(vm_area)) {
                vfree(stack);
                return -ENOMEM;
        }
        /*
         * We can't call find_vm_area() in interrupt context, and
         * free_thread_stack() can be called in interrupt context,
         * so cache the vm_struct.
         */
        tsk->stack_vm_area = vm_area;
        stack = kasan_reset_tag(stack);
        tsk->stack = stack;
        return 0;
}

static void free_thread_stack(struct task_struct *tsk)
{
        if (!try_release_thread_stack_to_cache(tsk->stack_vm_area))
                thread_stack_delayed_free(tsk);

        tsk->stack = NULL;
        tsk->stack_vm_area = NULL;
}

#else /* !CONFIG_VMAP_STACK */

/*
 * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
 * kmemcache based allocator.
 */
#if THREAD_SIZE >= PAGE_SIZE

static void thread_stack_free_rcu(struct rcu_head *rh)
{
        __free_pages(virt_to_page(rh), THREAD_SIZE_ORDER);
}

static void thread_stack_delayed_free(struct task_struct *tsk)
{
        struct rcu_head *rh = tsk->stack;

        call_rcu(rh, thread_stack_free_rcu);
}

static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
        struct page *page = alloc_pages_node(node, THREADINFO_GFP,
                                             THREAD_SIZE_ORDER);

        if (likely(page)) {
                tsk->stack = kasan_reset_tag(page_address(page));
                return 0;
        }
        return -ENOMEM;
}

static void free_thread_stack(struct task_struct *tsk)
{
        thread_stack_delayed_free(tsk);
        tsk->stack = NULL;
}

#else /* !(THREAD_SIZE >= PAGE_SIZE) */

static struct kmem_cache *thread_stack_cache;

static void thread_stack_free_rcu(struct rcu_head *rh)
{
        kmem_cache_free(thread_stack_cache, rh);
}

static void thread_stack_delayed_free(struct task_struct *tsk)
{
        struct rcu_head *rh = tsk->stack;

        call_rcu(rh, thread_stack_free_rcu);
}

static int alloc_thread_stack_node(struct task_struct *tsk, int node)
{
        unsigned long *stack;
        stack = kmem_cache_alloc_node(thread_stack_cache, THREADINFO_GFP, node);
        stack = kasan_reset_tag(stack);
        tsk->stack = stack;
        return stack ? 0 : -ENOMEM;
}

static void free_thread_stack(struct task_struct *tsk)
{
        thread_stack_delayed_free(tsk);
        tsk->stack = NULL;
}

void thread_stack_cache_init(void)
{
        thread_stack_cache = kmem_cache_create_usercopy("thread_stack",
                                        THREAD_SIZE, THREAD_SIZE, 0, 0,
                                        THREAD_SIZE, NULL);
        BUG_ON(thread_stack_cache == NULL);
}

#endif /* THREAD_SIZE >= PAGE_SIZE */
#endif /* CONFIG_VMAP_STACK */

/* SLAB cache for signal_struct structures (tsk->signal) */
static struct kmem_cache *signal_cachep;

/* SLAB cache for sighand_struct structures (tsk->sighand) */
struct kmem_cache *sighand_cachep;

/* SLAB cache for files_struct structures (tsk->files) */
struct kmem_cache *files_cachep;

/* SLAB cache for fs_struct structures (tsk->fs) */
struct kmem_cache *fs_cachep;

/* SLAB cache for mm_struct structures (tsk->mm) */
static struct kmem_cache *mm_cachep;

static void account_kernel_stack(struct task_struct *tsk, int account)
{
        if (IS_ENABLED(CONFIG_VMAP_STACK)) {
                struct vm_struct *vm_area = task_stack_vm_area(tsk);
                int i;

                for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
                        mod_lruvec_page_state(vm_area->pages[i], NR_KERNEL_STACK_KB,
                                              account * (PAGE_SIZE / 1024));
        } else {
                void *stack = task_stack_page(tsk);

                /* All stack pages are in the same node. */
                mod_lruvec_kmem_state(stack, NR_KERNEL_STACK_KB,
                                      account * (THREAD_SIZE / 1024));
        }
}

void exit_task_stack_account(struct task_struct *tsk)
{
        account_kernel_stack(tsk, -1);

        if (IS_ENABLED(CONFIG_VMAP_STACK)) {
                struct vm_struct *vm_area;
                int i;

                vm_area = task_stack_vm_area(tsk);
                for (i = 0; i < THREAD_SIZE / PAGE_SIZE; i++)
                        memcg_kmem_uncharge_page(vm_area->pages[i], 0);
        }
}

static void release_task_stack(struct task_struct *tsk)
{
        if (WARN_ON(READ_ONCE(tsk->__state) != TASK_DEAD))
                return;  /* Better to leak the stack than to free prematurely */

        free_thread_stack(tsk);
}

#ifdef CONFIG_THREAD_INFO_IN_TASK
void put_task_stack(struct task_struct *tsk)
{
        if (refcount_dec_and_test(&tsk->stack_refcount))
                release_task_stack(tsk);
}
#endif

void free_task(struct task_struct *tsk)
{
#ifdef CONFIG_SECCOMP
        WARN_ON_ONCE(tsk->seccomp.filter);
#endif
        release_user_cpus_ptr(tsk);
        scs_release(tsk);

#ifndef CONFIG_THREAD_INFO_IN_TASK
        /*
         * The task is finally done with both the stack and thread_info,
         * so free both.
         */
        release_task_stack(tsk);
#else
        /*
         * If the task had a separate stack allocation, it should be gone
         * by now.
         */
        WARN_ON_ONCE(refcount_read(&tsk->stack_refcount) != 0);
#endif
        rt_mutex_debug_task_free(tsk);
        ftrace_graph_exit_task(tsk);
        arch_release_task_struct(tsk);
        if (tsk->flags & PF_KTHREAD)
                free_kthread_struct(tsk);
        bpf_task_storage_free(tsk);
        free_task_struct(tsk);
}
EXPORT_SYMBOL(free_task);

void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm)
{
        struct file *exe_file;

        exe_file = get_mm_exe_file(oldmm);
        RCU_INIT_POINTER(mm->exe_file, exe_file);
        /*
         * We depend on the oldmm having properly denied write access to the
         * exe_file already.
         */
        if (exe_file && exe_file_deny_write_access(exe_file))
                pr_warn_once("exe_file_deny_write_access() failed in %s\n", __func__);
}

#ifdef CONFIG_MMU
static inline int mm_alloc_pgd(struct mm_struct *mm)
{
        mm->pgd = pgd_alloc(mm);
        if (unlikely(!mm->pgd))
                return -ENOMEM;
        return 0;
}

static inline void mm_free_pgd(struct mm_struct *mm)
{
        pgd_free(mm, mm->pgd);
}
#else
#define mm_alloc_pgd(mm)        (0)
#define mm_free_pgd(mm)
#endif /* CONFIG_MMU */

#ifdef CONFIG_MM_ID
static DEFINE_IDA(mm_ida);

static inline int mm_alloc_id(struct mm_struct *mm)
{
        int ret;

        ret = ida_alloc_range(&mm_ida, MM_ID_MIN, MM_ID_MAX, GFP_KERNEL);
        if (ret < 0)
                return ret;
        mm->mm_id = ret;
        return 0;
}

static inline void mm_free_id(struct mm_struct *mm)
{
        const mm_id_t id = mm->mm_id;

        mm->mm_id = MM_ID_DUMMY;
        if (id == MM_ID_DUMMY)
                return;
        if (WARN_ON_ONCE(id < MM_ID_MIN || id > MM_ID_MAX))
                return;
        ida_free(&mm_ida, id);
}
#else /* !CONFIG_MM_ID */
static inline int mm_alloc_id(struct mm_struct *mm) { return 0; }
static inline void mm_free_id(struct mm_struct *mm) {}
#endif /* CONFIG_MM_ID */

static void check_mm(struct mm_struct *mm)
{
        int i;

        BUILD_BUG_ON_MSG(ARRAY_SIZE(resident_page_types) != NR_MM_COUNTERS,
                         "Please make sure 'struct resident_page_types[]' is updated as well");

        for (i = 0; i < NR_MM_COUNTERS; i++) {
                long x = percpu_counter_sum(&mm->rss_stat[i]);

                if (unlikely(x)) {
                        pr_alert("BUG: Bad rss-counter state mm:%p type:%s val:%ld Comm:%s Pid:%d\n",
                                 mm, resident_page_types[i], x,
                                 current->comm,
                                 task_pid_nr(current));
                }
        }

        if (mm_pgtables_bytes(mm))
                pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n",
                                mm_pgtables_bytes(mm));

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
        VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
#endif
}

#define allocate_mm()        (kmem_cache_alloc(mm_cachep, GFP_KERNEL))
#define free_mm(mm)        (kmem_cache_free(mm_cachep, (mm)))

static void do_check_lazy_tlb(void *arg)
{
        struct mm_struct *mm = arg;

        WARN_ON_ONCE(current->active_mm == mm);
}

static void do_shoot_lazy_tlb(void *arg)
{
        struct mm_struct *mm = arg;

        if (current->active_mm == mm) {
                WARN_ON_ONCE(current->mm);
                current->active_mm = &init_mm;
                switch_mm(mm, &init_mm, current);
        }
}

static void cleanup_lazy_tlbs(struct mm_struct *mm)
{
        if (!IS_ENABLED(CONFIG_MMU_LAZY_TLB_SHOOTDOWN)) {
                /*
                 * In this case, lazy tlb mms are refounted and would not reach
                 * __mmdrop until all CPUs have switched away and mmdrop()ed.
                 */
                return;
        }

        /*
         * Lazy mm shootdown does not refcount "lazy tlb mm" usage, rather it
         * requires lazy mm users to switch to another mm when the refcount
         * drops to zero, before the mm is freed. This requires IPIs here to
         * switch kernel threads to init_mm.
         *
         * archs that use IPIs to flush TLBs can piggy-back that lazy tlb mm
         * switch with the final userspace teardown TLB flush which leaves the
         * mm lazy on this CPU but no others, reducing the need for additional
         * IPIs here. There are cases where a final IPI is still required here,
         * such as the final mmdrop being performed on a different CPU than the
         * one exiting, or kernel threads using the mm when userspace exits.
         *
         * IPI overheads have not found to be expensive, but they could be
         * reduced in a number of possible ways, for example (roughly
         * increasing order of complexity):
         * - The last lazy reference created by exit_mm() could instead switch
         *   to init_mm, however it's probable this will run on the same CPU
         *   immediately afterwards, so this may not reduce IPIs much.
         * - A batch of mms requiring IPIs could be gathered and freed at once.
         * - CPUs store active_mm where it can be remotely checked without a
         *   lock, to filter out false-positives in the cpumask.
         * - After mm_users or mm_count reaches zero, switching away from the
         *   mm could clear mm_cpumask to reduce some IPIs, perhaps together
         *   with some batching or delaying of the final IPIs.
         * - A delayed freeing and RCU-like quiescing sequence based on mm
         *   switching to avoid IPIs completely.
         */
        on_each_cpu_mask(mm_cpumask(mm), do_shoot_lazy_tlb, (void *)mm, 1);
        if (IS_ENABLED(CONFIG_DEBUG_VM_SHOOT_LAZIES))
                on_each_cpu(do_check_lazy_tlb, (void *)mm, 1);
}

/*
 * Called when the last reference to the mm
 * is dropped: either by a lazy thread or by
 * mmput. Free the page directory and the mm.
 */
void __mmdrop(struct mm_struct *mm)
{
        BUG_ON(mm == &init_mm);
        WARN_ON_ONCE(mm == current->mm);

        /* Ensure no CPUs are using this as their lazy tlb mm */
        cleanup_lazy_tlbs(mm);

        WARN_ON_ONCE(mm == current->active_mm);
        mm_free_pgd(mm);
        mm_free_id(mm);
        destroy_context(mm);
        mmu_notifier_subscriptions_destroy(mm);
        check_mm(mm);
        put_user_ns(mm->user_ns);
        mm_pasid_drop(mm);
        mm_destroy_cid(mm);
        percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);

        free_mm(mm);
}
EXPORT_SYMBOL_GPL(__mmdrop);

static void mmdrop_async_fn(struct work_struct *work)
{
        struct mm_struct *mm;

        mm = container_of(work, struct mm_struct, async_put_work);
        __mmdrop(mm);
}

static void mmdrop_async(struct mm_struct *mm)
{
        if (unlikely(atomic_dec_and_test(&mm->mm_count))) {
                INIT_WORK(&mm->async_put_work, mmdrop_async_fn);
                schedule_work(&mm->async_put_work);
        }
}

static inline void free_signal_struct(struct signal_struct *sig)
{
        taskstats_tgid_free(sig);
        sched_autogroup_exit(sig);
        /*
         * __mmdrop is not safe to call from softirq context on x86 due to
         * pgd_dtor so postpone it to the async context
         */
        if (sig->oom_mm)
                mmdrop_async(sig->oom_mm);
        kmem_cache_free(signal_cachep, sig);
}

static inline void put_signal_struct(struct signal_struct *sig)
{
        if (refcount_dec_and_test(&sig->sigcnt))
                free_signal_struct(sig);
}

void __put_task_struct(struct task_struct *tsk)
{
        WARN_ON(!tsk->exit_state);
        WARN_ON(refcount_read(&tsk->usage));
        WARN_ON(tsk == current);

        unwind_task_free(tsk);
        io_uring_free(tsk);
        cgroup_task_free(tsk);
        task_numa_free(tsk, true);
        security_task_free(tsk);
        exit_creds(tsk);
        delayacct_tsk_free(tsk);
        put_signal_struct(tsk->signal);
        sched_core_free(tsk);
        free_task(tsk);
}
EXPORT_SYMBOL_GPL(__put_task_struct);

void __put_task_struct_rcu_cb(struct rcu_head *rhp)
{
        struct task_struct *task = container_of(rhp, struct task_struct, rcu);

        __put_task_struct(task);
}
EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb);

void __init __weak arch_task_cache_init(void) { }

/*
 * set_max_threads
 */
static void __init set_max_threads(unsigned int max_threads_suggested)
{
        u64 threads;
        unsigned long nr_pages = memblock_estimated_nr_free_pages();

        /*
         * The number of threads shall be limited such that the thread
         * structures may only consume a small part of the available memory.
         */
        if (fls64(nr_pages) + fls64(PAGE_SIZE) > 64)
                threads = MAX_THREADS;
        else
                threads = div64_u64((u64) nr_pages * (u64) PAGE_SIZE,
                                    (u64) THREAD_SIZE * 8UL);

        if (threads > max_threads_suggested)
                threads = max_threads_suggested;

        max_threads = clamp_t(u64, threads, MIN_THREADS, MAX_THREADS);
}

#ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT
/* Initialized by the architecture: */
int arch_task_struct_size __read_mostly;
#endif

static void __init task_struct_whitelist(unsigned long *offset, unsigned long *size)
{
        /* Fetch thread_struct whitelist for the architecture. */
        arch_thread_struct_whitelist(offset, size);

        /*
         * Handle zero-sized whitelist or empty thread_struct, otherwise
         * adjust offset to position of thread_struct in task_struct.
         */
        if (unlikely(*size == 0))
                *offset = 0;
        else
                *offset += offsetof(struct task_struct, thread);
}

void __init fork_init(void)
{
        int i;
#ifndef ARCH_MIN_TASKALIGN
#define ARCH_MIN_TASKALIGN        0
#endif
        int align = max_t(int, L1_CACHE_BYTES, ARCH_MIN_TASKALIGN);
        unsigned long useroffset, usersize;

        /* create a slab on which task_structs can be allocated */
        task_struct_whitelist(&useroffset, &usersize);
        task_struct_cachep = kmem_cache_create_usercopy("task_struct",
                        arch_task_struct_size, align,
                        SLAB_PANIC|SLAB_ACCOUNT,
                        useroffset, usersize, NULL);

        /* do the arch specific task caches init */
        arch_task_cache_init();

        set_max_threads(MAX_THREADS);

        init_task.signal->rlim[RLIMIT_NPROC].rlim_cur = max_threads/2;
        init_task.signal->rlim[RLIMIT_NPROC].rlim_max = max_threads/2;
        init_task.signal->rlim[RLIMIT_SIGPENDING] =
                init_task.signal->rlim[RLIMIT_NPROC];

        for (i = 0; i < UCOUNT_COUNTS; i++)
                init_user_ns.ucount_max[i] = max_threads/2;

        set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_NPROC,      RLIM_INFINITY);
        set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MSGQUEUE,   RLIM_INFINITY);
        set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY);
        set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK,    RLIM_INFINITY);

#ifdef CONFIG_VMAP_STACK
        cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache",
                          NULL, free_vm_stack_cache);
#endif

        scs_init();

        lockdep_init_task(&init_task);
        uprobes_init();
}

int __weak arch_dup_task_struct(struct task_struct *dst,
                                               struct task_struct *src)
{
        *dst = *src;
        return 0;
}

void set_task_stack_end_magic(struct task_struct *tsk)
{
        unsigned long *stackend;

        stackend = end_of_stack(tsk);
        *stackend = STACK_END_MAGIC;        /* for overflow detection */
}

static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
{
        struct task_struct *tsk;
        int err;

        if (node == NUMA_NO_NODE)
                node = tsk_fork_get_node(orig);
        tsk = alloc_task_struct_node(node);
        if (!tsk)
                return NULL;

        err = arch_dup_task_struct(tsk, orig);
        if (err)
                goto free_tsk;

        err = alloc_thread_stack_node(tsk, node);
        if (err)
                goto free_tsk;

#ifdef CONFIG_THREAD_INFO_IN_TASK
        refcount_set(&tsk->stack_refcount, 1);
#endif
        account_kernel_stack(tsk, 1);

        err = scs_prepare(tsk, node);
        if (err)
                goto free_stack;

#ifdef CONFIG_SECCOMP
        /*
         * We must handle setting up seccomp filters once we're under
         * the sighand lock in case orig has changed between now and
         * then. Until then, filter must be NULL to avoid messing up
         * the usage counts on the error path calling free_task.
         */
        tsk->seccomp.filter = NULL;
#endif

        setup_thread_stack(tsk, orig);
        clear_user_return_notifier(tsk);
        clear_tsk_need_resched(tsk);
        set_task_stack_end_magic(tsk);
        clear_syscall_work_syscall_user_dispatch(tsk);

#ifdef CONFIG_STACKPROTECTOR
        tsk->stack_canary = get_random_canary();
#endif
        if (orig->cpus_ptr == &orig->cpus_mask)
                tsk->cpus_ptr = &tsk->cpus_mask;
        dup_user_cpus_ptr(tsk, orig, node);

        /*
         * One for the user space visible state that goes away when reaped.
         * One for the scheduler.
         */
        refcount_set(&tsk->rcu_users, 2);
        /* One for the rcu users */
        refcount_set(&tsk->usage, 1);
#ifdef CONFIG_BLK_DEV_IO_TRACE
        tsk->btrace_seq = 0;
#endif
        tsk->splice_pipe = NULL;
        tsk->task_frag.page = NULL;
        tsk->wake_q.next = NULL;
        tsk->worker_private = NULL;

        kcov_task_init(tsk);
        kmsan_task_create(tsk);
        kmap_local_fork(tsk);

#ifdef CONFIG_FAULT_INJECTION
        tsk->fail_nth = 0;
#endif

#ifdef CONFIG_BLK_CGROUP
        tsk->throttle_disk = NULL;
        tsk->use_memdelay = 0;
#endif

#ifdef CONFIG_ARCH_HAS_CPU_PASID
        tsk->pasid_activated = 0;
#endif

#ifdef CONFIG_MEMCG
        tsk->active_memcg = NULL;
#endif

#ifdef CONFIG_X86_BUS_LOCK_DETECT
        tsk->reported_split_lock = 0;
#endif

#ifdef CONFIG_SCHED_MM_CID
        tsk->mm_cid.cid = MM_CID_UNSET;
        tsk->mm_cid.active = 0;
        INIT_HLIST_NODE(&tsk->mm_cid.node);
#endif
        return tsk;

free_stack:
        exit_task_stack_account(tsk);
        free_thread_stack(tsk);
free_tsk:
        free_task_struct(tsk);
        return NULL;
}

__cacheline_aligned_in_smp DEFINE_SPINLOCK(mmlist_lock);

static unsigned long coredump_filter = MMF_DUMP_FILTER_DEFAULT;

static int __init coredump_filter_setup(char *s)
{
        if (kstrtoul(s, 0, &coredump_filter))
                return 0;
        coredump_filter <<= MMF_DUMP_FILTER_SHIFT;
        coredump_filter &= MMF_DUMP_FILTER_MASK;
        return 1;
}

__setup("coredump_filter=", coredump_filter_setup);

#include <linux/init_task.h>

static void mm_init_aio(struct mm_struct *mm)
{
#ifdef CONFIG_AIO
        spin_lock_init(&mm->ioctx_lock);
        mm->ioctx_table = NULL;
#endif
}

static __always_inline void mm_clear_owner(struct mm_struct *mm,
                                           struct task_struct *p)
{
#ifdef CONFIG_MEMCG
        if (mm->owner == p)
                WRITE_ONCE(mm->owner, NULL);
#endif
}

static void mm_init_owner(struct mm_struct *mm, struct task_struct *p)
{
#ifdef CONFIG_MEMCG
        mm->owner = p;
#endif
}

static void mm_init_uprobes_state(struct mm_struct *mm)
{
#ifdef CONFIG_UPROBES
        mm->uprobes_state.xol_area = NULL;
        arch_uprobe_init_state(mm);
#endif
}

static void mmap_init_lock(struct mm_struct *mm)
{
        init_rwsem(&mm->mmap_lock);
        mm_lock_seqcount_init(mm);
#ifdef CONFIG_PER_VMA_LOCK
        rcuwait_init(&mm->vma_writer_wait);
#endif
}

static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
        struct user_namespace *user_ns)
{
        mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
        mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
        atomic_set(&mm->mm_users, 1);
        atomic_set(&mm->mm_count, 1);
        seqcount_init(&mm->write_protect_seq);
        mmap_init_lock(mm);
        INIT_LIST_HEAD(&mm->mmlist);
        mm_pgtables_bytes_init(mm);
        mm->map_count = 0;
        mm->locked_vm = 0;
        atomic64_set(&mm->pinned_vm, 0);
        memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
        spin_lock_init(&mm->page_table_lock);
        spin_lock_init(&mm->arg_lock);
        mm_init_cpumask(mm);
        mm_init_aio(mm);
        mm_init_owner(mm, p);
        mm_pasid_init(mm);
        RCU_INIT_POINTER(mm->exe_file, NULL);
        mmu_notifier_subscriptions_init(mm);
        init_tlb_flush_pending(mm);
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
        mm->pmd_huge_pte = NULL;
#endif
        mm_init_uprobes_state(mm);
        hugetlb_count_init(mm);

        mm_flags_clear_all(mm);
        if (current->mm) {
                unsigned long flags = __mm_flags_get_word(current->mm);

                __mm_flags_overwrite_word(mm, mmf_init_legacy_flags(flags));
                mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK;
        } else {
                __mm_flags_overwrite_word(mm, coredump_filter);
                mm->def_flags = 0;
        }

        if (futex_mm_init(mm))
                goto fail_mm_init;

        if (mm_alloc_pgd(mm))
                goto fail_nopgd;

        if (mm_alloc_id(mm))
                goto fail_noid;

        if (init_new_context(p, mm))
                goto fail_nocontext;

        if (mm_alloc_cid(mm, p))
                goto fail_cid;

        if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
                                     NR_MM_COUNTERS))
                goto fail_pcpu;

        mm->user_ns = get_user_ns(user_ns);
        lru_gen_init_mm(mm);
        return mm;

fail_pcpu:
        mm_destroy_cid(mm);
fail_cid:
        destroy_context(mm);
fail_nocontext:
        mm_free_id(mm);
fail_noid:
        mm_free_pgd(mm);
fail_nopgd:
        futex_hash_free(mm);
fail_mm_init:
        free_mm(mm);
        return NULL;
}

/*
 * Allocate and initialize an mm_struct.
 */
struct mm_struct *mm_alloc(void)
{
        struct mm_struct *mm;

        mm = allocate_mm();
        if (!mm)
                return NULL;

        memset(mm, 0, sizeof(*mm));
        return mm_init(mm, current, current_user_ns());
}
EXPORT_SYMBOL_IF_KUNIT(mm_alloc);

static inline void __mmput(struct mm_struct *mm)
{
        VM_BUG_ON(atomic_read(&mm->mm_users));

        uprobe_clear_state(mm);
        exit_aio(mm);
        ksm_exit(mm);
        khugepaged_exit(mm); /* must run before exit_mmap */
        exit_mmap(mm);
        mm_put_huge_zero_folio(mm);
        set_mm_exe_file(mm, NULL);
        if (!list_empty(&mm->mmlist)) {
                spin_lock(&mmlist_lock);
                list_del(&mm->mmlist);
                spin_unlock(&mmlist_lock);
        }
        if (mm->binfmt)
                module_put(mm->binfmt->module);
        lru_gen_del_mm(mm);
        futex_hash_free(mm);
        mmdrop(mm);
}

/*
 * Decrement the use count and release all resources for an mm.
 */
void mmput(struct mm_struct *mm)
{
        might_sleep();

        if (atomic_dec_and_test(&mm->mm_users))
                __mmput(mm);
}
EXPORT_SYMBOL_GPL(mmput);

#if defined(CONFIG_MMU) || defined(CONFIG_FUTEX_PRIVATE_HASH)
static void mmput_async_fn(struct work_struct *work)
{
        struct mm_struct *mm = container_of(work, struct mm_struct,
                                            async_put_work);

        __mmput(mm);
}

void mmput_async(struct mm_struct *mm)
{
        if (atomic_dec_and_test(&mm->mm_users)) {
                INIT_WORK(&mm->async_put_work, mmput_async_fn);
                schedule_work(&mm->async_put_work);
        }
}
EXPORT_SYMBOL_GPL(mmput_async);
#endif

/**
 * set_mm_exe_file - change a reference to the mm's executable file
 * @mm: The mm to change.
 * @new_exe_file: The new file to use.
 *
 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
 *
 * Main users are mmput() and sys_execve(). Callers prevent concurrent
 * invocations: in mmput() nobody alive left, in execve it happens before
 * the new mm is made visible to anyone.
 *
 * Can only fail if new_exe_file != NULL.
 */
int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
{
        struct file *old_exe_file;

        /*
         * It is safe to dereference the exe_file without RCU as
         * this function is only called if nobody else can access
         * this mm -- see comment above for justification.
         */
        old_exe_file = rcu_dereference_raw(mm->exe_file);

        if (new_exe_file) {
                /*
                 * We expect the caller (i.e., sys_execve) to already denied
                 * write access, so this is unlikely to fail.
                 */
                if (unlikely(exe_file_deny_write_access(new_exe_file)))
                        return -EACCES;
                get_file(new_exe_file);
        }
        rcu_assign_pointer(mm->exe_file, new_exe_file);
        if (old_exe_file) {
                exe_file_allow_write_access(old_exe_file);
                fput(old_exe_file);
        }
        return 0;
}

/**
 * replace_mm_exe_file - replace a reference to the mm's executable file
 * @mm: The mm to change.
 * @new_exe_file: The new file to use.
 *
 * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
 *
 * Main user is sys_prctl(PR_SET_MM_MAP/EXE_FILE).
 */
int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file)
{
        struct vm_area_struct *vma;
        struct file *old_exe_file;
        int ret = 0;

        /* Forbid mm->exe_file change if old file still mapped. */
        old_exe_file = get_mm_exe_file(mm);
        if (old_exe_file) {
                VMA_ITERATOR(vmi, mm, 0);
                mmap_read_lock(mm);
                for_each_vma(vmi, vma) {
                        if (!vma->vm_file)
                                continue;
                        if (path_equal(&vma->vm_file->f_path,
                                       &old_exe_file->f_path)) {
                                ret = -EBUSY;
                                break;
                        }
                }
                mmap_read_unlock(mm);
                fput(old_exe_file);
                if (ret)
                        return ret;
        }

        ret = exe_file_deny_write_access(new_exe_file);
        if (ret)
                return -EACCES;
        get_file(new_exe_file);

        /* set the new file */
        mmap_write_lock(mm);
        old_exe_file = rcu_dereference_raw(mm->exe_file);
        rcu_assign_pointer(mm->exe_file, new_exe_file);
        mmap_write_unlock(mm);

        if (old_exe_file) {
                exe_file_allow_write_access(old_exe_file);
                fput(old_exe_file);
        }
        return 0;
}

/**
 * get_mm_exe_file - acquire a reference to the mm's executable file
 * @mm: The mm of interest.
 *
 * Returns %NULL if mm has no associated executable file.
 * User must release file via fput().
 */
struct file *get_mm_exe_file(struct mm_struct *mm)
{
        struct file *exe_file;

        rcu_read_lock();
        exe_file = get_file_rcu(&mm->exe_file);
        rcu_read_unlock();
        return exe_file;
}

/**
 * get_task_exe_file - acquire a reference to the task's executable file
 * @task: The task.
 *
 * Returns %NULL if task's mm (if any) has no associated executable file or
 * this is a kernel thread with borrowed mm (see the comment above get_task_mm).
 * User must release file via fput().
 */
struct file *get_task_exe_file(struct task_struct *task)
{
        struct file *exe_file = NULL;
        struct mm_struct *mm;

        if (task->flags & PF_KTHREAD)
                return NULL;

        task_lock(task);
        mm = task->mm;
        if (mm)
                exe_file = get_mm_exe_file(mm);
        task_unlock(task);
        return exe_file;
}

/**
 * get_task_mm - acquire a reference to the task's mm
 * @task: The task.
 *
 * Returns %NULL if the task has no mm.  Checks PF_KTHREAD (meaning
 * this kernel workthread has transiently adopted a user mm with kthread_use_mm,
 * to do its AIO) is not set and if so returns a reference to it, after
 * bumping up the use count.  User must release the mm via mmput()
 * after use.  Typically used by /proc and ptrace.
 */
struct mm_struct *get_task_mm(struct task_struct *task)
{
        struct mm_struct *mm;

        if (task->flags & PF_KTHREAD)
                return NULL;

        task_lock(task);
        mm = task->mm;
        if (mm)
                mmget(mm);
        task_unlock(task);
        return mm;
}
EXPORT_SYMBOL_GPL(get_task_mm);

static bool may_access_mm(struct mm_struct *mm, struct task_struct *task, unsigned int mode)
{
        if (mm == current->mm)
                return true;
        if (ptrace_may_access(task, mode))
                return true;
        if ((mode & PTRACE_MODE_READ) && perfmon_capable())
                return true;
        return false;
}

struct mm_struct *mm_access(struct task_struct *task, unsigned int mode)
{
        struct mm_struct *mm;
        int err;

        err =  down_read_killable(&task->signal->exec_update_lock);
        if (err)
                return ERR_PTR(err);

        mm = get_task_mm(task);
        if (!mm) {
                mm = ERR_PTR(-ESRCH);
        } else if (!may_access_mm(mm, task, mode)) {
                mmput(mm);
                mm = ERR_PTR(-EACCES);
        }
        up_read(&task->signal->exec_update_lock);

        return mm;
}

static void complete_vfork_done(struct task_struct *tsk)
{
        struct completion *vfork;

        task_lock(tsk);
        vfork = tsk->vfork_done;
        if (likely(vfork)) {
                tsk->vfork_done = NULL;
                complete(vfork);
        }
        task_unlock(tsk);
}

static int wait_for_vfork_done(struct task_struct *child,
                                struct completion *vfork)
{
        unsigned int state = TASK_KILLABLE|TASK_FREEZABLE;
        int killed;

        cgroup_enter_frozen();
        killed = wait_for_completion_state(vfork, state);
        cgroup_leave_frozen(false);

        if (killed) {
                task_lock(child);
                child->vfork_done = NULL;
                task_unlock(child);
        }

        put_task_struct(child);
        return killed;
}

/* Please note the differences between mmput and mm_release.
 * mmput is called whenever we stop holding onto a mm_struct,
 * error success whatever.
 *
 * mm_release is called after a mm_struct has been removed
 * from the current process.
 *
 * This difference is important for error handling, when we
 * only half set up a mm_struct for a new process and need to restore
 * the old one.  Because we mmput the new mm_struct before
 * restoring the old one. . .
 * Eric Biederman 10 January 1998
 */
static void mm_release(struct task_struct *tsk, struct mm_struct *mm)
{
        uprobe_free_utask(tsk);

        /* Get rid of any cached register state */
        deactivate_mm(tsk, mm);

        /*
         * Signal userspace if we're not exiting with a core dump
         * because we want to leave the value intact for debugging
         * purposes.
         */
        if (tsk->clear_child_tid) {
                if (atomic_read(&mm->mm_users) > 1) {
                        /*
                         * We don't check the error code - if userspace has
                         * not set up a proper pointer then tough luck.
                         */
                        put_user(0, tsk->clear_child_tid);
                        do_futex(tsk->clear_child_tid, FUTEX_WAKE,
                                        1, NULL, NULL, 0, 0);
                }
                tsk->clear_child_tid = NULL;
        }

        /*
         * All done, finally we can wake up parent and return this mm to him.
         * Also kthread_stop() uses this completion for synchronization.
         */
        if (tsk->vfork_done)
                complete_vfork_done(tsk);
}

void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm)
{
        futex_exit_release(tsk);
        mm_release(tsk, mm);
}

void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm)
{
        futex_exec_release(tsk);
        mm_release(tsk, mm);
}

/**
 * dup_mm() - duplicates an existing mm structure
 * @tsk: the task_struct with which the new mm will be associated.
 * @oldmm: the mm to duplicate.
 *
 * Allocates a new mm structure and duplicates the provided @oldmm structure
 * content into it.
 *
 * Return: the duplicated mm or NULL on failure.
 */
static struct mm_struct *dup_mm(struct task_struct *tsk,
                                struct mm_struct *oldmm)
{
        struct mm_struct *mm;
        int err;

        mm = allocate_mm();
        if (!mm)
                goto fail_nomem;

        memcpy(mm, oldmm, sizeof(*mm));

        if (!mm_init(mm, tsk, mm->user_ns))
                goto fail_nomem;

        uprobe_start_dup_mmap();
        err = dup_mmap(mm, oldmm);
        if (err)
                goto free_pt;
        uprobe_end_dup_mmap();

        mm->hiwater_rss = get_mm_rss(mm);
        mm->hiwater_vm = mm->total_vm;

        if (mm->binfmt && !try_module_get(mm->binfmt->module))
                goto free_pt;

        return mm;

free_pt:
        /* don't put binfmt in mmput, we haven't got module yet */
        mm->binfmt = NULL;
        mm_init_owner(mm, NULL);
        mmput(mm);
        if (err)
                uprobe_end_dup_mmap();

fail_nomem:
        return NULL;
}

static int copy_mm(u64 clone_flags, struct task_struct *tsk)
{
        struct mm_struct *mm, *oldmm;

        tsk->min_flt = tsk->maj_flt = 0;
        tsk->nvcsw = tsk->nivcsw = 0;
#ifdef CONFIG_DETECT_HUNG_TASK
        tsk->last_switch_count = tsk->nvcsw + tsk->nivcsw;
        tsk->last_switch_time = 0;
#endif

        tsk->mm = NULL;
        tsk->active_mm = NULL;

        /*
         * Are we cloning a kernel thread?
         *
         * We need to steal a active VM for that..
         */
        oldmm = current->mm;
        if (!oldmm)
                return 0;

        if (clone_flags & CLONE_VM) {
                mmget(oldmm);
                mm = oldmm;
        } else {
                mm = dup_mm(tsk, current->mm);
                if (!mm)
                        return -ENOMEM;
        }

        tsk->mm = mm;
        tsk->active_mm = mm;
        return 0;
}

static int copy_fs(u64 clone_flags, struct task_struct *tsk)
{
        struct fs_struct *fs = current->fs;
        if (clone_flags & CLONE_FS) {
                /* tsk->fs is already what we want */
                read_seqlock_excl(&fs->seq);
                /* "users" and "in_exec" locked for check_unsafe_exec() */
                if (fs->in_exec) {
                        read_sequnlock_excl(&fs->seq);
                        return -EAGAIN;
                }
                fs->users++;
                read_sequnlock_excl(&fs->seq);
                return 0;
        }
        tsk->fs = copy_fs_struct(fs);
        if (!tsk->fs)
                return -ENOMEM;
        return 0;
}

static int copy_files(u64 clone_flags, struct task_struct *tsk,
                      int no_files)
{
        struct files_struct *oldf, *newf;

        /*
         * A background process may not have any files ...
         */
        oldf = current->files;
        if (!oldf)
                return 0;

        if (no_files) {
                tsk->files = NULL;
                return 0;
        }

        if (clone_flags & CLONE_FILES) {
                atomic_inc(&oldf->count);
                return 0;
        }

        newf = dup_fd(oldf, NULL);
        if (IS_ERR(newf))
                return PTR_ERR(newf);

        tsk->files = newf;
        return 0;
}

static int copy_sighand(u64 clone_flags, struct task_struct *tsk)
{
        struct sighand_struct *sig;

        if (clone_flags & CLONE_SIGHAND) {
                refcount_inc(&current->sighand->count);
                return 0;
        }
        sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL);
        RCU_INIT_POINTER(tsk->sighand, sig);
        if (!sig)
                return -ENOMEM;

        refcount_set(&sig->count, 1);
        spin_lock_irq(&current->sighand->siglock);
        memcpy(sig->action, current->sighand->action, sizeof(sig->action));
        spin_unlock_irq(&current->sighand->siglock);

        /* Reset all signal handler not set to SIG_IGN to SIG_DFL. */
        if (clone_flags & CLONE_CLEAR_SIGHAND)
                flush_signal_handlers(tsk, 0);

        return 0;
}

void __cleanup_sighand(struct sighand_struct *sighand)
{
        if (refcount_dec_and_test(&sighand->count)) {
                signalfd_cleanup(sighand);
                /*
                 * sighand_cachep is SLAB_TYPESAFE_BY_RCU so we can free it
                 * without an RCU grace period, see __lock_task_sighand().
                 */
                kmem_cache_free(sighand_cachep, sighand);
        }
}

/*
 * Initialize POSIX timer handling for a thread group.
 */
static void posix_cpu_timers_init_group(struct signal_struct *sig)
{
        struct posix_cputimers *pct = &sig->posix_cputimers;
        unsigned long cpu_limit;

        cpu_limit = READ_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur);
        posix_cputimers_group_init(pct, cpu_limit);
}

static int copy_signal(u64 clone_flags, struct task_struct *tsk)
{
        struct signal_struct *sig;

        if (clone_flags & CLONE_THREAD)
                return 0;

        sig = kmem_cache_zalloc(signal_cachep, GFP_KERNEL);
        tsk->signal = sig;
        if (!sig)
                return -ENOMEM;

        sig->nr_threads = 1;
        sig->quick_threads = 1;
        atomic_set(&sig->live, 1);
        refcount_set(&sig->sigcnt, 1);

        /* list_add(thread_node, thread_head) without INIT_LIST_HEAD() */
        sig->thread_head = (struct list_head)LIST_HEAD_INIT(tsk->thread_node);
        tsk->thread_node = (struct list_head)LIST_HEAD_INIT(sig->thread_head);

        init_waitqueue_head(&sig->wait_chldexit);
        sig->curr_target = tsk;
        init_sigpending(&sig->shared_pending);
        INIT_HLIST_HEAD(&sig->multiprocess);
        seqlock_init(&sig->stats_lock);
        prev_cputime_init(&sig->prev_cputime);

#ifdef CONFIG_POSIX_TIMERS
        INIT_HLIST_HEAD(&sig->posix_timers);
        INIT_HLIST_HEAD(&sig->ignored_posix_timers);
        hrtimer_setup(&sig->real_timer, it_real_fn, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
#endif

        task_lock(current->group_leader);
        memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
        task_unlock(current->group_leader);

        posix_cpu_timers_init_group(sig);

        tty_audit_fork(sig);
        sched_autogroup_fork(sig);

#ifdef CONFIG_CGROUPS
        init_rwsem(&sig->cgroup_threadgroup_rwsem);
#endif

        sig->oom_score_adj = current->signal->oom_score_adj;
        sig->oom_score_adj_min = current->signal->oom_score_adj_min;

        mutex_init(&sig->cred_guard_mutex);
        init_rwsem(&sig->exec_update_lock);

        return 0;
}

static void copy_seccomp(struct task_struct *p)
{
#ifdef CONFIG_SECCOMP
        /*
         * Must be called with sighand->lock held, which is common to
         * all threads in the group. Holding cred_guard_mutex is not
         * needed because this new task is not yet running and cannot
         * be racing exec.
         */
        assert_spin_locked(&current->sighand->siglock);

        /* Ref-count the new filter user, and assign it. */
        get_seccomp_filter(current);
        p->seccomp = current->seccomp;

        /*
         * Explicitly enable no_new_privs here in case it got set
         * between the task_struct being duplicated and holding the
         * sighand lock. The seccomp state and nnp must be in sync.
         */
        if (task_no_new_privs(current))
                task_set_no_new_privs(p);

        /*
         * If the parent gained a seccomp mode after copying thread
         * flags and between before we held the sighand lock, we have
         * to manually enable the seccomp thread flag here.
         */
        if (p->seccomp.mode != SECCOMP_MODE_DISABLED)
                set_task_syscall_work(p, SECCOMP);
#endif
}

SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
{
        current->clear_child_tid = tidptr;

        return task_pid_vnr(current);
}

static void rt_mutex_init_task(struct task_struct *p)
{
        raw_spin_lock_init(&p->pi_lock);
#ifdef CONFIG_RT_MUTEXES
        p->pi_waiters = RB_ROOT_CACHED;
        p->pi_top_task = NULL;
        p->pi_blocked_on = NULL;
#endif
}

static inline void init_task_pid_links(struct task_struct *task)
{
        enum pid_type type;

        for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type)
                INIT_HLIST_NODE(&task->pid_links[type]);
}

static inline void
init_task_pid(struct task_struct *task, enum pid_type type, struct pid *pid)
{
        if (type == PIDTYPE_PID)
                task->thread_pid = pid;
        else
                task->signal->pids[type] = pid;
}

static inline void rcu_copy_process(struct task_struct *p)
{
#ifdef CONFIG_PREEMPT_RCU
        p->rcu_read_lock_nesting = 0;
        p->rcu_read_unlock_special.s = 0;
        p->rcu_blocked_node = NULL;
        INIT_LIST_HEAD(&p->rcu_node_entry);
#endif /* #ifdef CONFIG_PREEMPT_RCU */
#ifdef CONFIG_TASKS_RCU
        p->rcu_tasks_holdout = false;
        INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
        p->rcu_tasks_idle_cpu = -1;
        INIT_LIST_HEAD(&p->rcu_tasks_exit_list);
#endif /* #ifdef CONFIG_TASKS_RCU */
#ifdef CONFIG_TASKS_TRACE_RCU
        p->trc_reader_nesting = 0;
#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
}

/**
 * pidfd_prepare - allocate a new pidfd_file and reserve a pidfd
 * @pid:   the struct pid for which to create a pidfd
 * @flags: flags of the new @pidfd
 * @ret_file: return the new pidfs file
 *
 * Allocate a new file that stashes @pid and reserve a new pidfd number in the
 * caller's file descriptor table. The pidfd is reserved but not installed yet.
 *
 * The helper verifies that @pid is still in use, without PIDFD_THREAD the
 * task identified by @pid must be a thread-group leader.
 *
 * If this function returns successfully the caller is responsible to either
 * call fd_install() passing the returned pidfd and pidfd file as arguments in
 * order to install the pidfd into its file descriptor table or they must use
 * put_unused_fd() and fput() on the returned pidfd and pidfd file
 * respectively.
 *
 * This function is useful when a pidfd must already be reserved but there
 * might still be points of failure afterwards and the caller wants to ensure
 * that no pidfd is leaked into its file descriptor table.
 *
 * Return: On success, a reserved pidfd is returned from the function and a new
 *         pidfd file is returned in the last argument to the function. On
 *         error, a negative error code is returned from the function and the
 *         last argument remains unchanged.
 */
int pidfd_prepare(struct pid *pid, unsigned int flags, struct file **ret_file)
{
        struct file *pidfs_file;

        /*
         * PIDFD_STALE is only allowed to be passed if the caller knows
         * that @pid is already registered in pidfs and thus
         * PIDFD_INFO_EXIT information is guaranteed to be available.
         */
        if (!(flags & PIDFD_STALE)) {
                /*
                 * While holding the pidfd waitqueue lock removing the
                 * task linkage for the thread-group leader pid
                 * (PIDTYPE_TGID) isn't possible. Thus, if there's still
                 * task linkage for PIDTYPE_PID not having thread-group
                 * leader linkage for the pid means it wasn't a
                 * thread-group leader in the first place.
                 */
                guard(spinlock_irq)(&pid->wait_pidfd.lock);

                /* Task has already been reaped. */
                if (!pid_has_task(pid, PIDTYPE_PID))
                        return -ESRCH;
                /*
                 * If this struct pid isn't used as a thread-group
                 * leader but the caller requested to create a
                 * thread-group leader pidfd then report ENOENT.
                 */
                if (!(flags & PIDFD_THREAD) && !pid_has_task(pid, PIDTYPE_TGID))
                        return -ENOENT;
        }

        CLASS(get_unused_fd, pidfd)(O_CLOEXEC);
        if (pidfd < 0)
                return pidfd;

        pidfs_file = pidfs_alloc_file(pid, flags | O_RDWR);
        if (IS_ERR(pidfs_file))
                return PTR_ERR(pidfs_file);

        *ret_file = pidfs_file;
        return take_fd(pidfd);
}

static void __delayed_free_task(struct rcu_head *rhp)
{
        struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);

        free_task(tsk);
}

static __always_inline void delayed_free_task(struct task_struct *tsk)
{
        if (IS_ENABLED(CONFIG_MEMCG))
                call_rcu(&tsk->rcu, __delayed_free_task);
        else
                free_task(tsk);
}

static void copy_oom_score_adj(u64 clone_flags, struct task_struct *tsk)
{
        /* Skip if kernel thread */
        if (!tsk->mm)
                return;

        /* Skip if spawning a thread or using vfork */
        if ((clone_flags & (CLONE_VM | CLONE_THREAD | CLONE_VFORK)) != CLONE_VM)
                return;

        /* We need to synchronize with __set_oom_adj */
        mutex_lock(&oom_adj_mutex);
        mm_flags_set(MMF_MULTIPROCESS, tsk->mm);
        /* Update the values in case they were changed after copy_signal */
        tsk->signal->oom_score_adj = current->signal->oom_score_adj;
        tsk->signal->oom_score_adj_min = current->signal->oom_score_adj_min;
        mutex_unlock(&oom_adj_mutex);
}

#ifdef CONFIG_RV
static void rv_task_fork(struct task_struct *p)
{
        memset(&p->rv, 0, sizeof(p->rv));
}
#else
#define rv_task_fork(p) do {} while (0)
#endif

static bool need_futex_hash_allocate_default(u64 clone_flags)
{
        /*
         * Allocate a default futex hash for any sibling that will
         * share the parent's mm, except vfork.
         */
        return (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM;
}

/*
 * This creates a new process as a copy of the old one,
 * but does not actually start it yet.
 *
 * It copies the registers, and all the appropriate
 * parts of the process environment (as per the clone
 * flags). The actual kick-off is left to the caller.
 */
__latent_entropy struct task_struct *copy_process(
                                        struct pid *pid,
                                        int trace,
                                        int node,
                                        struct kernel_clone_args *args)
{
        int pidfd = -1, retval;
        struct task_struct *p;
        struct multiprocess_signals delayed;
        struct file *pidfile = NULL;
        const u64 clone_flags = args->flags;
        struct nsproxy *nsp = current->nsproxy;

        /*
         * Don't allow sharing the root directory with processes in a different
         * namespace
         */
        if ((clone_flags & (CLONE_NEWNS|CLONE_FS)) == (CLONE_NEWNS|CLONE_FS))
                return ERR_PTR(-EINVAL);

        if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
                return ERR_PTR(-EINVAL);

        /*
         * Thread groups must share signals as well, and detached threads
         * can only be started up within the thread group.
         */
        if ((clone_flags & CLONE_THREAD) && !(clone_flags & CLONE_SIGHAND))
                return ERR_PTR(-EINVAL);

        /*
         * Shared signal handlers imply shared VM. By way of the above,
         * thread groups also imply shared VM. Blocking this case allows
         * for various simplifications in other code.
         */
        if ((clone_flags & CLONE_SIGHAND) && !(clone_flags & CLONE_VM))
                return ERR_PTR(-EINVAL);

        /*
         * Siblings of global init remain as zombies on exit since they are
         * not reaped by their parent (swapper). To solve this and to avoid
         * multi-rooted process trees, prevent global and container-inits
         * from creating siblings.
         */
        if ((clone_flags & CLONE_PARENT) &&
                                current->signal->flags & SIGNAL_UNKILLABLE)
                return ERR_PTR(-EINVAL);

        /*
         * If the new process will be in a different pid or user namespace
         * do not allow it to share a thread group with the forking task.
         */
        if (clone_flags & CLONE_THREAD) {
                if ((clone_flags & (CLONE_NEWUSER | CLONE_NEWPID)) ||
                    (task_active_pid_ns(current) != nsp->pid_ns_for_children))
                        return ERR_PTR(-EINVAL);
        }

        if (clone_flags & CLONE_PIDFD) {
                /*
                 * - CLONE_DETACHED is blocked so that we can potentially
                 *   reuse it later for CLONE_PIDFD.
                 */
                if (clone_flags & CLONE_DETACHED)
                        return ERR_PTR(-EINVAL);
        }

        if (clone_flags & CLONE_AUTOREAP) {
                if (clone_flags & CLONE_THREAD)
                        return ERR_PTR(-EINVAL);
                if (clone_flags & CLONE_PARENT)
                        return ERR_PTR(-EINVAL);
                if (args->exit_signal)
                        return ERR_PTR(-EINVAL);
        }

        if ((clone_flags & CLONE_PARENT) && current->signal->autoreap)
                return ERR_PTR(-EINVAL);

        if (clone_flags & CLONE_NNP) {
                if (clone_flags & CLONE_THREAD)
                        return ERR_PTR(-EINVAL);
        }

        if (clone_flags & CLONE_PIDFD_AUTOKILL) {
                if (!(clone_flags & CLONE_PIDFD))
                        return ERR_PTR(-EINVAL);
                if (!(clone_flags & CLONE_AUTOREAP))
                        return ERR_PTR(-EINVAL);
                if (clone_flags & CLONE_THREAD)
                        return ERR_PTR(-EINVAL);
                /*
                 * Without CLONE_NNP the child could escalate privileges
                 * after being spawned, so require CAP_SYS_ADMIN.
                 * With CLONE_NNP the child can't gain new privileges,
                 * so allow unprivileged usage.
                 */
                if (!(clone_flags & CLONE_NNP) &&
                    !ns_capable(current_user_ns(), CAP_SYS_ADMIN))
                        return ERR_PTR(-EPERM);
        }

        /*
         * Force any signals received before this point to be delivered
         * before the fork happens.  Collect up signals sent to multiple
         * processes that happen during the fork and delay them so that
         * they appear to happen after the fork.
         */
        sigemptyset(&delayed.signal);
        INIT_HLIST_NODE(&delayed.node);

        spin_lock_irq(&current->sighand->siglock);
        if (!(clone_flags & CLONE_THREAD))
                hlist_add_head(&delayed.node, &current->signal->multiprocess);
        recalc_sigpending();
        spin_unlock_irq(&current->sighand->siglock);
        retval = -ERESTARTNOINTR;
        if (task_sigpending(current))
                goto fork_out;

        retval = -ENOMEM;
        p = dup_task_struct(current, node);
        if (!p)
                goto fork_out;
        p->flags &= ~PF_KTHREAD;
        if (args->kthread)
                p->flags |= PF_KTHREAD;
        if (args->user_worker) {
                /*
                 * Mark us a user worker, and block any signal that isn't
                 * fatal or STOP
                 */
                p->flags |= PF_USER_WORKER;
                siginitsetinv(&p->blocked, sigmask(SIGKILL)|sigmask(SIGSTOP));
        }
        if (args->io_thread)
                p->flags |= PF_IO_WORKER;

        if (args->name)
                strscpy_pad(p->comm, args->name, sizeof(p->comm));

        p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? args->child_tid : NULL;
        /*
         * TID is cleared in mm_release() when the task exits
         */
        p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? args->child_tid : NULL;

        ftrace_graph_init_task(p);

        rt_mutex_init_task(p);
        raw_spin_lock_init(&p->blocked_lock);

        lockdep_assert_irqs_enabled();
#ifdef CONFIG_PROVE_LOCKING
        DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
#endif
        retval = copy_creds(p, clone_flags);
        if (retval < 0)
                goto bad_fork_free;

        retval = -EAGAIN;
        if (is_rlimit_overlimit(task_ucounts(p), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) {
                if (p->real_cred->user != INIT_USER &&
                    !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN))
                        goto bad_fork_cleanup_count;
        }
        current->flags &= ~PF_NPROC_EXCEEDED;

        /*
         * If multiple threads are within copy_process(), then this check
         * triggers too late. This doesn't hurt, the check is only there
         * to stop root fork bombs.
         */
        retval = -EAGAIN;
        if (data_race(nr_threads >= max_threads))
                goto bad_fork_cleanup_count;

        delayacct_tsk_init(p);        /* Must remain after dup_task_struct() */
        p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE | PF_NO_SETAFFINITY);
        p->flags |= PF_FORKNOEXEC;
        INIT_LIST_HEAD(&p->children);
        INIT_LIST_HEAD(&p->sibling);
        rcu_copy_process(p);
        p->vfork_done = NULL;
        spin_lock_init(&p->alloc_lock);

        init_sigpending(&p->pending);

        p->utime = p->stime = p->gtime = 0;
#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
        p->utimescaled = p->stimescaled = 0;
#endif
        prev_cputime_init(&p->prev_cputime);

#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
        seqcount_init(&p->vtime.seqcount);
        p->vtime.starttime = 0;
        p->vtime.state = VTIME_INACTIVE;
#endif

#ifdef CONFIG_IO_URING
        p->io_uring = NULL;
        retval = io_uring_fork(p);
        if (unlikely(retval))
                goto bad_fork_cleanup_delayacct;
        retval = -EAGAIN;
#endif

        p->default_timer_slack_ns = current->timer_slack_ns;

#ifdef CONFIG_PSI
        p->psi_flags = 0;
#endif

        task_io_accounting_init(&p->ioac);
        acct_clear_integrals(p);

        posix_cputimers_init(&p->posix_cputimers);
        tick_dep_init_task(p);

        p->io_context = NULL;
        audit_set_context(p, NULL);
        cgroup_fork(p);
        if (args->kthread) {
                if (!set_kthread_struct(p))
                        goto bad_fork_cleanup_delayacct;
        }
#ifdef CONFIG_NUMA
        p->mempolicy = mpol_dup(p->mempolicy);
        if (IS_ERR(p->mempolicy)) {
                retval = PTR_ERR(p->mempolicy);
                p->mempolicy = NULL;
                goto bad_fork_cleanup_delayacct;
        }
#endif
#ifdef CONFIG_CPUSETS
        p->cpuset_mem_spread_rotor = NUMA_NO_NODE;
        seqcount_spinlock_init(&p->mems_allowed_seq, &p->alloc_lock);
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
        memset(&p->irqtrace, 0, sizeof(p->irqtrace));
        p->irqtrace.hardirq_disable_ip        = _THIS_IP_;
        p->irqtrace.softirq_enable_ip        = _THIS_IP_;
        p->softirqs_enabled                = 1;
        p->softirq_context                = 0;
#endif

        p->pagefault_disabled = 0;

        lockdep_init_task(p);

        p->blocked_on = NULL; /* not blocked yet */

#ifdef CONFIG_BCACHE
        p->sequential_io        = 0;
        p->sequential_io_avg        = 0;
#endif
#ifdef CONFIG_BPF_SYSCALL
        RCU_INIT_POINTER(p->bpf_storage, NULL);
        p->bpf_ctx = NULL;
#endif

        unwind_task_init(p);

        /* Perform scheduler related setup. Assign this task to a CPU. */
        retval = sched_fork(clone_flags, p);
        if (retval)
                goto bad_fork_cleanup_policy;

        retval = perf_event_init_task(p, clone_flags);
        if (retval)
                goto bad_fork_sched_cancel_fork;
        retval = audit_alloc(p);
        if (retval)
                goto bad_fork_cleanup_perf;
        /* copy all the process information */
        shm_init_task(p);
        retval = security_task_alloc(p, clone_flags);
        if (retval)
                goto bad_fork_cleanup_audit;
        retval = copy_semundo(clone_flags, p);
        if (retval)
                goto bad_fork_cleanup_security;
        retval = copy_files(clone_flags, p, args->no_files);
        if (retval)
                goto bad_fork_cleanup_semundo;
        retval = copy_fs(clone_flags, p);
        if (retval)
                goto bad_fork_cleanup_files;
        retval = copy_sighand(clone_flags, p);
        if (retval)
                goto bad_fork_cleanup_fs;
        retval = copy_signal(clone_flags, p);
        if (retval)
                goto bad_fork_cleanup_sighand;
        retval = copy_mm(clone_flags, p);
        if (retval)
                goto bad_fork_cleanup_signal;
        retval = copy_namespaces(clone_flags, p);
        if (retval)
                goto bad_fork_cleanup_mm;
        retval = copy_io(clone_flags, p);
        if (retval)
                goto bad_fork_cleanup_namespaces;
        retval = copy_thread(p, args);
        if (retval)
                goto bad_fork_cleanup_io;

        stackleak_task_init(p);

        if (pid != &init_struct_pid) {
                pid = alloc_pid(p->nsproxy->pid_ns_for_children, args->set_tid,
                                args->set_tid_size);
                if (IS_ERR(pid)) {
                        retval = PTR_ERR(pid);
                        goto bad_fork_cleanup_thread;
                }
        }

        /*
         * This has to happen after we've potentially unshared the file
         * descriptor table (so that the pidfd doesn't leak into the child
         * if the fd table isn't shared).
         */
        if (clone_flags & CLONE_PIDFD) {
                unsigned flags = PIDFD_STALE;

                if (clone_flags & CLONE_THREAD)
                        flags |= PIDFD_THREAD;
                if (clone_flags & CLONE_PIDFD_AUTOKILL)
                        flags |= PIDFD_AUTOKILL;

                /*
                 * Note that no task has been attached to @pid yet indicate
                 * that via CLONE_PIDFD.
                 */
                retval = pidfd_prepare(pid, flags, &pidfile);
                if (retval < 0)
                        goto bad_fork_free_pid;
                pidfd = retval;

                retval = put_user(pidfd, args->pidfd);
                if (retval)
                        goto bad_fork_put_pidfd;
        }

#ifdef CONFIG_BLOCK
        p->plug = NULL;
#endif
        futex_init_task(p);

        /*
         * sigaltstack should be cleared when sharing the same VM
         */
        if ((clone_flags & (CLONE_VM|CLONE_VFORK)) == CLONE_VM)
                sas_ss_reset(p);

        /*
         * Syscall tracing and stepping should be turned off in the
         * child regardless of CLONE_PTRACE.
         */
        user_disable_single_step(p);
        clear_task_syscall_work(p, SYSCALL_TRACE);
#if defined(CONFIG_GENERIC_ENTRY) || defined(TIF_SYSCALL_EMU)
        clear_task_syscall_work(p, SYSCALL_EMU);
#endif
        clear_tsk_latency_tracing(p);

        /* ok, now we should be set up.. */
        p->pid = pid_nr(pid);
        if (clone_flags & CLONE_THREAD) {
                p->group_leader = current->group_leader;
                p->tgid = current->tgid;
        } else {
                p->group_leader = p;
                p->tgid = p->pid;
        }

        p->nr_dirtied = 0;
        p->nr_dirtied_pause = 128 >> (PAGE_SHIFT - 10);
        p->dirty_paused_when = 0;

        p->pdeath_signal = 0;
        p->task_works = NULL;
        clear_posix_cputimers_work(p);

#ifdef CONFIG_KRETPROBES
        p->kretprobe_instances.first = NULL;
#endif
#ifdef CONFIG_RETHOOK
        p->rethooks.first = NULL;
#endif

        /*
         * Ensure that the cgroup subsystem policies allow the new process to be
         * forked. It should be noted that the new process's css_set can be changed
         * between here and cgroup_post_fork() if an organisation operation is in
         * progress.
         */
        retval = cgroup_can_fork(p, args);
        if (retval)
                goto bad_fork_put_pidfd;

        /*
         * Now that the cgroups are pinned, re-clone the parent cgroup and put
         * the new task on the correct runqueue. All this *before* the task
         * becomes visible.
         *
         * This isn't part of ->can_fork() because while the re-cloning is
         * cgroup specific, it unconditionally needs to place the task on a
         * runqueue.
         */
        retval = sched_cgroup_fork(p, args);
        if (retval)
                goto bad_fork_cancel_cgroup;

        if (need_futex_hash_allocate_default(clone_flags)) {
                retval = futex_hash_allocate_default();
                if (retval)
                        goto bad_fork_cancel_cgroup;
                /*
                 * If we fail beyond this point we don't free the allocated
                 * futex hash map. We assume that another thread will be created
                 * and makes use of it. The hash map will be freed once the main
                 * thread terminates.
                 */
        }
        /*
         * From this point on we must avoid any synchronous user-space
         * communication until we take the tasklist-lock. In particular, we do
         * not want user-space to be able to predict the process start-time by
         * stalling fork(2) after we recorded the start_time but before it is
         * visible to the system.
         */

        p->start_time = ktime_get_ns();
        p->start_boottime = ktime_get_boottime_ns();

        /*
         * Make it visible to the rest of the system, but dont wake it up yet.
         * Need tasklist lock for parent etc handling!
         */
        write_lock_irq(&tasklist_lock);

        /* CLONE_PARENT re-uses the old parent */
        if (clone_flags & (CLONE_PARENT|CLONE_THREAD)) {
                p->real_parent = current->real_parent;
                p->parent_exec_id = current->parent_exec_id;
                if (clone_flags & CLONE_THREAD)
                        p->exit_signal = -1;
                else
                        p->exit_signal = current->group_leader->exit_signal;
        } else {
                p->real_parent = current;
                p->parent_exec_id = current->self_exec_id;
                p->exit_signal = args->exit_signal;
        }

        klp_copy_process(p);

        sched_core_fork(p);

        spin_lock(&current->sighand->siglock);

        rv_task_fork(p);

        rseq_fork(p, clone_flags);

        /*
         * If zap_pid_ns_processes() was called after alloc_pid(), the new
         * child missed SIGKILL.  If current is not in the same namespace,
         * we can't rely on fatal_signal_pending() below.
         */
        if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
                retval = -ENOMEM;
                goto bad_fork_core_free;
        }

        /* Let kill terminate clone/fork in the middle */
        if (fatal_signal_pending(current)) {
                retval = -EINTR;
                goto bad_fork_core_free;
        }

        /* No more failure paths after this point. */

        /*
         * Copy seccomp details explicitly here, in case they were changed
         * before holding sighand lock.
         */
        copy_seccomp(p);

        if (clone_flags & CLONE_NNP)
                task_set_no_new_privs(p);

        init_task_pid_links(p);
        if (likely(p->pid)) {
                ptrace_init_task(p, (clone_flags & CLONE_PTRACE) || trace);

                init_task_pid(p, PIDTYPE_PID, pid);
                if (thread_group_leader(p)) {
                        init_task_pid(p, PIDTYPE_TGID, pid);
                        init_task_pid(p, PIDTYPE_PGID, task_pgrp(current));
                        init_task_pid(p, PIDTYPE_SID, task_session(current));

                        if (is_child_reaper(pid)) {
                                struct pid_namespace *ns = ns_of_pid(pid);

                                ASSERT_EXCLUSIVE_WRITER(ns->child_reaper);
                                WRITE_ONCE(ns->child_reaper, p);
                                p->signal->flags |= SIGNAL_UNKILLABLE;
                        }
                        p->signal->shared_pending.signal = delayed.signal;
                        p->signal->tty = tty_kref_get(current->signal->tty);
                        /*
                         * Inherit has_child_subreaper flag under the same
                         * tasklist_lock with adding child to the process tree
                         * for propagate_has_child_subreaper optimization.
                         */
                        p->signal->has_child_subreaper = p->real_parent->signal->has_child_subreaper ||
                                                         p->real_parent->signal->is_child_subreaper;
                        if (clone_flags & CLONE_AUTOREAP)
                                p->signal->autoreap = 1;
                        list_add_tail(&p->sibling, &p->real_parent->children);
                        list_add_tail_rcu(&p->tasks, &init_task.tasks);
                        attach_pid(p, PIDTYPE_TGID);
                        attach_pid(p, PIDTYPE_PGID);
                        attach_pid(p, PIDTYPE_SID);
                        __this_cpu_inc(process_counts);
                } else {
                        current->signal->nr_threads++;
                        current->signal->quick_threads++;
                        atomic_inc(&current->signal->live);
                        refcount_inc(&current->signal->sigcnt);
                        task_join_group_stop(p);
                        list_add_tail_rcu(&p->thread_node,
                                          &p->signal->thread_head);
                }
                attach_pid(p, PIDTYPE_PID);
                nr_threads++;
        }
        total_forks++;
        hlist_del_init(&delayed.node);
        spin_unlock(&current->sighand->siglock);
        syscall_tracepoint_update(p);
        write_unlock_irq(&tasklist_lock);

        if (pidfile)
                fd_install(pidfd, pidfile);

        proc_fork_connector(p);
        /*
         * sched_ext needs @p to be associated with its cgroup in its post_fork
         * hook. cgroup_post_fork() should come before sched_post_fork().
         */
        cgroup_post_fork(p, args);
        sched_post_fork(p);
        perf_event_fork(p);

        trace_task_newtask(p, clone_flags);
        uprobe_copy_process(p, clone_flags);
        user_events_fork(p, clone_flags);

        copy_oom_score_adj(clone_flags, p);

        return p;

bad_fork_core_free:
        sched_core_free(p);
        spin_unlock(&current->sighand->siglock);
        write_unlock_irq(&tasklist_lock);
bad_fork_cancel_cgroup:
        cgroup_cancel_fork(p, args);
bad_fork_put_pidfd:
        if (clone_flags & CLONE_PIDFD) {
                fput(pidfile);
                put_unused_fd(pidfd);
        }
bad_fork_free_pid:
        if (pid != &init_struct_pid)
                free_pid(pid);
bad_fork_cleanup_thread:
        exit_thread(p);
bad_fork_cleanup_io:
        if (p->io_context)
                exit_io_context(p);
bad_fork_cleanup_namespaces:
        exit_nsproxy_namespaces(p);
bad_fork_cleanup_mm:
        if (p->mm) {
                mm_clear_owner(p->mm, p);
                mmput(p->mm);
        }
bad_fork_cleanup_signal:
        if (!(clone_flags & CLONE_THREAD))
                free_signal_struct(p->signal);
bad_fork_cleanup_sighand:
        __cleanup_sighand(p->sighand);
bad_fork_cleanup_fs:
        exit_fs(p); /* blocking */
bad_fork_cleanup_files:
        exit_files(p); /* blocking */
bad_fork_cleanup_semundo:
        exit_sem(p);
bad_fork_cleanup_security:
        security_task_free(p);
bad_fork_cleanup_audit:
        audit_free(p);
bad_fork_cleanup_perf:
        perf_event_free_task(p);
bad_fork_sched_cancel_fork:
        sched_cancel_fork(p);
bad_fork_cleanup_policy:
        lockdep_free_task(p);
#ifdef CONFIG_NUMA
        mpol_put(p->mempolicy);
#endif
bad_fork_cleanup_delayacct:
        io_uring_free(p);
        delayacct_tsk_free(p);
bad_fork_cleanup_count:
        dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);
        exit_cred_namespaces(p);
        exit_creds(p);
bad_fork_free:
        WRITE_ONCE(p->__state, TASK_DEAD);
        exit_task_stack_account(p);
        put_task_stack(p);
        delayed_free_task(p);
fork_out:
        spin_lock_irq(&current->sighand->siglock);
        hlist_del_init(&delayed.node);
        spin_unlock_irq(&current->sighand->siglock);
        return ERR_PTR(retval);
}

static inline void init_idle_pids(struct task_struct *idle)
{
        enum pid_type type;

        for (type = PIDTYPE_PID; type < PIDTYPE_MAX; ++type) {
                INIT_HLIST_NODE(&idle->pid_links[type]); /* not really needed */
                init_task_pid(idle, type, &init_struct_pid);
        }
}

static int idle_dummy(void *dummy)
{
        /* This function is never called */
        return 0;
}

struct task_struct * __init fork_idle(int cpu)
{
        struct task_struct *task;
        struct kernel_clone_args args = {
                .flags                = CLONE_VM,
                .fn                = &idle_dummy,
                .fn_arg                = NULL,
                .kthread        = 1,
                .idle                = 1,
        };

        task = copy_process(&init_struct_pid, 0, cpu_to_node(cpu), &args);
        if (!IS_ERR(task)) {
                init_idle_pids(task);
                init_idle(task, cpu);
        }

        return task;
}

/*
 * This is like kernel_clone(), but shaved down and tailored to just
 * creating io_uring workers. It returns a created task, or an error pointer.
 * The returned task is inactive, and the caller must fire it up through
 * wake_up_new_task(p). All signals are blocked in the created task.
 */
struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node)
{
        unsigned long flags = CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|
                              CLONE_IO|CLONE_VM|CLONE_UNTRACED;
        struct kernel_clone_args args = {
                .flags                = flags,
                .fn                = fn,
                .fn_arg                = arg,
                .io_thread        = 1,
                .user_worker        = 1,
        };

        return copy_process(NULL, 0, node, &args);
}

/*
 *  Ok, this is the main fork-routine.
 *
 * It copies the process, and if successful kick-starts
 * it and waits for it to finish using the VM if required.
 */
pid_t kernel_clone(struct kernel_clone_args *args)
{
        u64 clone_flags = args->flags;
        struct completion vfork;
        struct pid *pid;
        struct task_struct *p;
        int trace = 0;
        pid_t nr;

        /*
         * Creating an empty mount namespace implies creating a new mount
         * namespace.  Set this before copy_process() so that the
         * CLONE_NEWNS|CLONE_FS mutual exclusion check works correctly.
         */
        if (clone_flags & CLONE_EMPTY_MNTNS) {
                clone_flags |= CLONE_NEWNS;
                args->flags = clone_flags;
        }

        /*
         * For legacy clone() calls, CLONE_PIDFD uses the parent_tid argument
         * to return the pidfd. Hence, CLONE_PIDFD and CLONE_PARENT_SETTID are
         * mutually exclusive. With clone3() CLONE_PIDFD has grown a separate
         * field in struct clone_args and it still doesn't make sense to have
         * them both point at the same memory location. Performing this check
         * here has the advantage that we don't need to have a separate helper
         * to check for legacy clone().
         */
        if ((clone_flags & CLONE_PIDFD) &&
            (clone_flags & CLONE_PARENT_SETTID) &&
            (args->pidfd == args->parent_tid))
                return -EINVAL;

        if (!valid_signal(args->exit_signal))
                return -EINVAL;

        /*
         * Determine whether and which event to report to ptracer.  When
         * called from kernel_thread or CLONE_UNTRACED is explicitly
         * requested, no event is reported; otherwise, report if the event
         * for the type of forking is enabled.
         */
        if (!(clone_flags & CLONE_UNTRACED)) {
                if (clone_flags & CLONE_VFORK)
                        trace = PTRACE_EVENT_VFORK;
                else if (args->exit_signal != SIGCHLD)
                        trace = PTRACE_EVENT_CLONE;
                else
                        trace = PTRACE_EVENT_FORK;

                if (likely(!ptrace_event_enabled(current, trace)))
                        trace = 0;
        }

        p = copy_process(NULL, trace, NUMA_NO_NODE, args);
        add_latent_entropy();

        if (IS_ERR(p))
                return PTR_ERR(p);

        /*
         * Do this prior waking up the new thread - the thread pointer
         * might get invalid after that point, if the thread exits quickly.
         */
        trace_sched_process_fork(current, p);

        pid = get_task_pid(p, PIDTYPE_PID);
        nr = pid_vnr(pid);

        if (clone_flags & CLONE_PARENT_SETTID)
                put_user(nr, args->parent_tid);

        if (clone_flags & CLONE_VFORK) {
                p->vfork_done = &vfork;
                init_completion(&vfork);
                get_task_struct(p);
        }

        if (IS_ENABLED(CONFIG_LRU_GEN_WALKS_MMU) && !(clone_flags & CLONE_VM)) {
                /* lock the task to synchronize with memcg migration */
                task_lock(p);
                lru_gen_add_mm(p->mm);
                task_unlock(p);
        }

        wake_up_new_task(p);

        /* forking complete and child started to run, tell ptracer */
        if (unlikely(trace))
                ptrace_event_pid(trace, pid);

        if (clone_flags & CLONE_VFORK) {
                if (!wait_for_vfork_done(p, &vfork))
                        ptrace_event_pid(PTRACE_EVENT_VFORK_DONE, pid);
        }

        put_pid(pid);
        return nr;
}

/*
 * Create a kernel thread.
 */
pid_t kernel_thread(int (*fn)(void *), void *arg, const char *name,
                    unsigned long flags)
{
        struct kernel_clone_args args = {
                .flags                = ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL),
                .exit_signal        = (flags & CSIGNAL),
                .fn                = fn,
                .fn_arg                = arg,
                .name                = name,
                .kthread        = 1,
        };

        return kernel_clone(&args);
}

/*
 * Create a user mode thread.
 */
pid_t user_mode_thread(int (*fn)(void *), void *arg, unsigned long flags)
{
        struct kernel_clone_args args = {
                .flags                = ((flags | CLONE_VM | CLONE_UNTRACED) & ~CSIGNAL),
                .exit_signal        = (flags & CSIGNAL),
                .fn                = fn,
                .fn_arg                = arg,
        };

        return kernel_clone(&args);
}

#ifdef __ARCH_WANT_SYS_FORK
SYSCALL_DEFINE0(fork)
{
#ifdef CONFIG_MMU
        struct kernel_clone_args args = {
                .exit_signal = SIGCHLD,
        };

        return kernel_clone(&args);
#else
        /* can not support in nommu mode */
        return -EINVAL;
#endif
}
#endif

#ifdef __ARCH_WANT_SYS_VFORK
SYSCALL_DEFINE0(vfork)
{
        struct kernel_clone_args args = {
                .flags                = CLONE_VFORK | CLONE_VM,
                .exit_signal        = SIGCHLD,
        };

        return kernel_clone(&args);
}
#endif

#ifdef __ARCH_WANT_SYS_CLONE
#ifdef CONFIG_CLONE_BACKWARDS
SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
                 int __user *, parent_tidptr,
                 unsigned long, tls,
                 int __user *, child_tidptr)
#elif defined(CONFIG_CLONE_BACKWARDS2)
SYSCALL_DEFINE5(clone, unsigned long, newsp, unsigned long, clone_flags,
                 int __user *, parent_tidptr,
                 int __user *, child_tidptr,
                 unsigned long, tls)
#elif defined(CONFIG_CLONE_BACKWARDS3)
SYSCALL_DEFINE6(clone, unsigned long, clone_flags, unsigned long, newsp,
                int, stack_size,
                int __user *, parent_tidptr,
                int __user *, child_tidptr,
                unsigned long, tls)
#else
SYSCALL_DEFINE5(clone, unsigned long, clone_flags, unsigned long, newsp,
                 int __user *, parent_tidptr,
                 int __user *, child_tidptr,
                 unsigned long, tls)
#endif
{
        struct kernel_clone_args args = {
                .flags                = (lower_32_bits(clone_flags) & ~CSIGNAL),
                .pidfd                = parent_tidptr,
                .child_tid        = child_tidptr,
                .parent_tid        = parent_tidptr,
                .exit_signal        = (lower_32_bits(clone_flags) & CSIGNAL),
                .stack                = newsp,
                .tls                = tls,
        };

        return kernel_clone(&args);
}
#endif

static noinline int copy_clone_args_from_user(struct kernel_clone_args *kargs,
                                              struct clone_args __user *uargs,
                                              size_t usize)
{
        int err;
        struct clone_args args;
        pid_t *kset_tid = kargs->set_tid;

        BUILD_BUG_ON(offsetofend(struct clone_args, tls) !=
                     CLONE_ARGS_SIZE_VER0);
        BUILD_BUG_ON(offsetofend(struct clone_args, set_tid_size) !=
                     CLONE_ARGS_SIZE_VER1);
        BUILD_BUG_ON(offsetofend(struct clone_args, cgroup) !=
                     CLONE_ARGS_SIZE_VER2);
        BUILD_BUG_ON(sizeof(struct clone_args) != CLONE_ARGS_SIZE_VER2);

        if (unlikely(usize > PAGE_SIZE))
                return -E2BIG;
        if (unlikely(usize < CLONE_ARGS_SIZE_VER0))
                return -EINVAL;

        err = copy_struct_from_user(&args, sizeof(args), uargs, usize);
        if (err)
                return err;

        if (unlikely(args.set_tid_size > MAX_PID_NS_LEVEL))
                return -EINVAL;

        if (unlikely(!args.set_tid && args.set_tid_size > 0))
                return -EINVAL;

        if (unlikely(args.set_tid && args.set_tid_size == 0))
                return -EINVAL;

        /*
         * Verify that higher 32bits of exit_signal are unset
         */
        if (unlikely(args.exit_signal & ~((u64)CSIGNAL)))
                return -EINVAL;

        if ((args.flags & CLONE_INTO_CGROUP) &&
            (args.cgroup > INT_MAX || usize < CLONE_ARGS_SIZE_VER2))
                return -EINVAL;

        *kargs = (struct kernel_clone_args){
                .flags                = args.flags,
                .pidfd                = u64_to_user_ptr(args.pidfd),
                .child_tid        = u64_to_user_ptr(args.child_tid),
                .parent_tid        = u64_to_user_ptr(args.parent_tid),
                .exit_signal        = args.exit_signal,
                .stack                = args.stack,
                .stack_size        = args.stack_size,
                .tls                = args.tls,
                .set_tid_size        = args.set_tid_size,
                .cgroup                = args.cgroup,
        };

        if (args.set_tid &&
                copy_from_user(kset_tid, u64_to_user_ptr(args.set_tid),
                        (kargs->set_tid_size * sizeof(pid_t))))
                return -EFAULT;

        kargs->set_tid = kset_tid;

        return 0;
}

/**
 * clone3_stack_valid - check and prepare stack
 * @kargs: kernel clone args
 *
 * Verify that the stack arguments userspace gave us are sane.
 * In addition, set the stack direction for userspace since it's easy for us to
 * determine.
 */
static inline bool clone3_stack_valid(struct kernel_clone_args *kargs)
{
        if (kargs->stack == 0) {
                if (kargs->stack_size > 0)
                        return false;
        } else {
                if (kargs->stack_size == 0)
                        return false;

                if (!access_ok((void __user *)kargs->stack, kargs->stack_size))
                        return false;

#if !defined(CONFIG_STACK_GROWSUP)
                kargs->stack += kargs->stack_size;
#endif
        }

        return true;
}

static bool clone3_args_valid(struct kernel_clone_args *kargs)
{
        /* Verify that no unknown flags are passed along. */
        if (kargs->flags &
            ~(CLONE_LEGACY_FLAGS | CLONE_CLEAR_SIGHAND |
              CLONE_INTO_CGROUP | CLONE_AUTOREAP | CLONE_NNP |
              CLONE_PIDFD_AUTOKILL | CLONE_EMPTY_MNTNS))
                return false;

        /*
         * - make the CLONE_DETACHED bit reusable for clone3
         * - make the CSIGNAL bits reusable for clone3
         */
        if (kargs->flags & (CLONE_DETACHED | (CSIGNAL & (~CLONE_NEWTIME))))
                return false;

        if ((kargs->flags & (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND)) ==
            (CLONE_SIGHAND | CLONE_CLEAR_SIGHAND))
                return false;

        if ((kargs->flags & (CLONE_THREAD | CLONE_PARENT)) &&
            kargs->exit_signal)
                return false;

        if (!clone3_stack_valid(kargs))
                return false;

        return true;
}

/**
 * sys_clone3 - create a new process with specific properties
 * @uargs: argument structure
 * @size:  size of @uargs
 *
 * clone3() is the extensible successor to clone()/clone2().
 * It takes a struct as argument that is versioned by its size.
 *
 * Return: On success, a positive PID for the child process.
 *         On error, a negative errno number.
 */
SYSCALL_DEFINE2(clone3, struct clone_args __user *, uargs, size_t, size)
{
        int err;

        struct kernel_clone_args kargs;
        pid_t set_tid[MAX_PID_NS_LEVEL];

#ifdef __ARCH_BROKEN_SYS_CLONE3
#warning clone3() entry point is missing, please fix
        return -ENOSYS;
#endif

        kargs.set_tid = set_tid;

        err = copy_clone_args_from_user(&kargs, uargs, size);
        if (err)
                return err;

        if (!clone3_args_valid(&kargs))
                return -EINVAL;

        return kernel_clone(&kargs);
}

void walk_process_tree(struct task_struct *top, proc_visitor visitor, void *data)
{
        struct task_struct *leader, *parent, *child;
        int res;

        read_lock(&tasklist_lock);
        leader = top = top->group_leader;
down:
        for_each_thread(leader, parent) {
                list_for_each_entry(child, &parent->children, sibling) {
                        res = visitor(child, data);
                        if (res) {
                                if (res < 0)
                                        goto out;
                                leader = child;
                                goto down;
                        }
up:
                        ;
                }
        }

        if (leader != top) {
                child = leader;
                parent = child->real_parent;
                leader = parent->group_leader;
                goto up;
        }
out:
        read_unlock(&tasklist_lock);
}

#ifndef ARCH_MIN_MMSTRUCT_ALIGN
#define ARCH_MIN_MMSTRUCT_ALIGN 0
#endif

static void sighand_ctor(void *data)
{
        struct sighand_struct *sighand = data;

        spin_lock_init(&sighand->siglock);
        init_waitqueue_head(&sighand->signalfd_wqh);
}

void __init mm_cache_init(void)
{
        unsigned int mm_size;

        /*
         * The mm_cpumask is located at the end of mm_struct, and is
         * dynamically sized based on the maximum CPU number this system
         * can have, taking hotplug into account (nr_cpu_ids).
         */
        mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size();

        mm_cachep = kmem_cache_create_usercopy("mm_struct",
                        mm_size, ARCH_MIN_MMSTRUCT_ALIGN,
                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
                        offsetof(struct mm_struct, saved_auxv),
                        sizeof_field(struct mm_struct, saved_auxv),
                        NULL);
}

void __init proc_caches_init(void)
{
        sighand_cachep = kmem_cache_create("sighand_cache",
                        sizeof(struct sighand_struct), 0,
                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
                        SLAB_ACCOUNT, sighand_ctor);
        signal_cachep = kmem_cache_create("signal_cache",
                        sizeof(struct signal_struct), 0,
                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
                        NULL);
        files_cachep = kmem_cache_create("files_cache",
                        sizeof(struct files_struct), 0,
                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
                        NULL);
        fs_cachep = kmem_cache_create("fs_cache",
                        sizeof(struct fs_struct), 0,
                        SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
                        NULL);
        mmap_init();
        nsproxy_cache_init();
}

/*
 * Check constraints on flags passed to the unshare system call.
 */
static int check_unshare_flags(unsigned long unshare_flags)
{
        if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_SIGHAND|
                                CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
                                CLONE_NS_ALL | UNSHARE_EMPTY_MNTNS))
                return -EINVAL;
        /*
         * Not implemented, but pretend it works if there is nothing
         * to unshare.  Note that unsharing the address space or the
         * signal handlers also need to unshare the signal queues (aka
         * CLONE_THREAD).
         */
        if (unshare_flags & (CLONE_THREAD | CLONE_SIGHAND | CLONE_VM)) {
                if (!thread_group_empty(current))
                        return -EINVAL;
        }
        if (unshare_flags & (CLONE_SIGHAND | CLONE_VM)) {
                if (refcount_read(&current->sighand->count) > 1)
                        return -EINVAL;
        }
        if (unshare_flags & CLONE_VM) {
                if (!current_is_single_threaded())
                        return -EINVAL;
        }

        return 0;
}

/*
 * Unshare the filesystem structure if it is being shared
 */
static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp)
{
        struct fs_struct *fs = current->fs;

        if (!(unshare_flags & CLONE_FS) || !fs)
                return 0;

        /* don't need lock here; in the worst case we'll do useless copy */
        if (!(unshare_flags & CLONE_NEWNS) && fs->users == 1)
                return 0;

        *new_fsp = copy_fs_struct(fs);
        if (!*new_fsp)
                return -ENOMEM;

        return 0;
}

/*
 * Unshare file descriptor table if it is being shared
 */
static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp)
{
        struct files_struct *fd = current->files;

        if ((unshare_flags & CLONE_FILES) &&
            (fd && atomic_read(&fd->count) > 1)) {
                fd = dup_fd(fd, NULL);
                if (IS_ERR(fd))
                        return PTR_ERR(fd);
                *new_fdp = fd;
        }

        return 0;
}

/*
 * unshare allows a process to 'unshare' part of the process
 * context which was originally shared using clone.  copy_*
 * functions used by kernel_clone() cannot be used here directly
 * because they modify an inactive task_struct that is being
 * constructed. Here we are modifying the current, active,
 * task_struct.
 */
int ksys_unshare(unsigned long unshare_flags)
{
        struct fs_struct *fs, *new_fs = NULL;
        struct files_struct *new_fd = NULL;
        struct cred *new_cred = NULL;
        struct nsproxy *new_nsproxy = NULL;
        int do_sysvsem = 0;
        int err;

        /*
         * If unsharing a user namespace must also unshare the thread group
         * and unshare the filesystem root and working directories.
         */
        if (unshare_flags & CLONE_NEWUSER)
                unshare_flags |= CLONE_THREAD | CLONE_FS;
        /*
         * If unsharing vm, must also unshare signal handlers.
         */
        if (unshare_flags & CLONE_VM)
                unshare_flags |= CLONE_SIGHAND;
        /*
         * If unsharing a signal handlers, must also unshare the signal queues.
         */
        if (unshare_flags & CLONE_SIGHAND)
                unshare_flags |= CLONE_THREAD;
        /*
         * If unsharing namespace, must also unshare filesystem information.
         */
        if (unshare_flags & UNSHARE_EMPTY_MNTNS)
                unshare_flags |= CLONE_NEWNS;
        if (unshare_flags & CLONE_NEWNS)
                unshare_flags |= CLONE_FS;

        err = check_unshare_flags(unshare_flags);
        if (err)
                goto bad_unshare_out;
        /*
         * CLONE_NEWIPC must also detach from the undolist: after switching
         * to a new ipc namespace, the semaphore arrays from the old
         * namespace are unreachable.
         */
        if (unshare_flags & (CLONE_NEWIPC|CLONE_SYSVSEM))
                do_sysvsem = 1;
        err = unshare_fs(unshare_flags, &new_fs);
        if (err)
                goto bad_unshare_out;
        err = unshare_fd(unshare_flags, &new_fd);
        if (err)
                goto bad_unshare_cleanup_fs;
        err = unshare_userns(unshare_flags, &new_cred);
        if (err)
                goto bad_unshare_cleanup_fd;
        err = unshare_nsproxy_namespaces(unshare_flags, &new_nsproxy,
                                         new_cred, new_fs);
        if (err)
                goto bad_unshare_cleanup_cred;
        if (new_cred) {
                err = set_cred_ucounts(new_cred);
                if (err)
                        goto bad_unshare_cleanup_nsproxy;
        }

        if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) {
                if (do_sysvsem) {
                        /*
                         * CLONE_SYSVSEM is equivalent to sys_exit().
                         */
                        exit_sem(current);
                }
                if (unshare_flags & CLONE_NEWIPC) {
                        /* Orphan segments in old ns (see sem above). */
                        exit_shm(current);
                        shm_init_task(current);
                }

                if (new_nsproxy) {
                        switch_task_namespaces(current, new_nsproxy);
                        new_nsproxy = NULL;
                }

                task_lock(current);

                if (new_fs) {
                        fs = current->fs;
                        read_seqlock_excl(&fs->seq);
                        current->fs = new_fs;
                        if (--fs->users)
                                new_fs = NULL;
                        else
                                new_fs = fs;
                        read_sequnlock_excl(&fs->seq);
                }

                if (new_fd)
                        swap(current->files, new_fd);

                task_unlock(current);

                if (new_cred) {
                        /* Install the new user namespace */
                        commit_creds(new_cred);
                        new_cred = NULL;
                }
        }

        perf_event_namespaces(current);

bad_unshare_cleanup_nsproxy:
        if (new_nsproxy)
                put_nsproxy(new_nsproxy);
bad_unshare_cleanup_cred:
        if (new_cred)
                put_cred(new_cred);
bad_unshare_cleanup_fd:
        if (new_fd)
                put_files_struct(new_fd);
bad_unshare_cleanup_fs:
        if (new_fs)
                free_fs_struct(new_fs);

bad_unshare_out:
        return err;
}

SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
{
        return ksys_unshare(unshare_flags);
}

/*
 *        Helper to unshare the files of the current task.
 *        We don't want to expose copy_files internals to
 *        the exec layer of the kernel.
 */

int unshare_files(void)
{
        struct task_struct *task = current;
        struct files_struct *old, *copy = NULL;
        int error;

        error = unshare_fd(CLONE_FILES, &copy);
        if (error || !copy)
                return error;

        old = task->files;
        task_lock(task);
        task->files = copy;
        task_unlock(task);
        put_files_struct(old);
        return 0;
}

static int sysctl_max_threads(const struct ctl_table *table, int write,
                       void *buffer, size_t *lenp, loff_t *ppos)
{
        struct ctl_table t;
        int ret;
        int threads = max_threads;
        int min = 1;
        int max = MAX_THREADS;

        t = *table;
        t.data = &threads;
        t.extra1 = &min;
        t.extra2 = &max;

        ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
        if (ret || !write)
                return ret;

        max_threads = threads;

        return 0;
}

static const struct ctl_table fork_sysctl_table[] = {
        {
                .procname        = "threads-max",
                .data                = NULL,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = sysctl_max_threads,
        },
};

static int __init init_fork_sysctl(void)
{
        register_sysctl_init("kernel", fork_sysctl_table);
        return 0;
}

subsys_initcall(init_fork_sysctl);






















































































































   43 
















    3 




    1 


    2 










   32 






    1 

    3 







    1 



   11 

    6 







    2 







   12 
    5 




    2 

    1 

    1 





    3 

    2 







   22 





    1 
    5 




    3 
















   20 









   20 


   20 

   19 
    1 











    4 




    3 
    1 




   18 










   18 
   18 





    5 
    8 



    5 



   18 








   18 


   18 




















   20 









   20 





    8 





    4 


    5 





    3 















    8 


    4 


    5 


    3 




























   30 


   30 






























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (c) 2014 Jiri Pirko <jiri@resnulli.us>
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/if_vlan.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>

#include <linux/tc_act/tc_vlan.h>
#include <net/tc_act/tc_vlan.h>

static struct tc_action_ops act_vlan_ops;

TC_INDIRECT_SCOPE int tcf_vlan_act(struct sk_buff *skb,
                                   const struct tc_action *a,
                                   struct tcf_result *res)
{
        struct tcf_vlan *v = to_vlan(a);
        struct tcf_vlan_params *p;
        int err;
        u16 tci;

        tcf_lastuse_update(&v->tcf_tm);
        tcf_action_update_bstats(&v->common, skb);

        /* Ensure 'data' points at mac_header prior calling vlan manipulating
         * functions.
         */
        if (skb_at_tc_ingress(skb))
                skb_push_rcsum(skb, skb->mac_len);

        p = rcu_dereference_bh(v->vlan_p);

        switch (p->tcfv_action) {
        case TCA_VLAN_ACT_POP:
                err = skb_vlan_pop(skb);
                if (err)
                        goto drop;
                break;
        case TCA_VLAN_ACT_PUSH:
                err = skb_vlan_push(skb, p->tcfv_push_proto, p->tcfv_push_vid |
                                    (p->tcfv_push_prio << VLAN_PRIO_SHIFT));
                if (err)
                        goto drop;
                break;
        case TCA_VLAN_ACT_MODIFY:
                /* No-op if no vlan tag (either hw-accel or in-payload) */
                if (!skb_vlan_tagged(skb))
                        goto out;
                /* extract existing tag (and guarantee no hw-accel tag) */
                if (skb_vlan_tag_present(skb)) {
                        tci = skb_vlan_tag_get(skb);
                        __vlan_hwaccel_clear_tag(skb);
                } else {
                        /* in-payload vlan tag, pop it */
                        err = __skb_vlan_pop(skb, &tci);
                        if (err)
                                goto drop;
                }
                /* replace the vid */
                tci = (tci & ~VLAN_VID_MASK) | p->tcfv_push_vid;
                /* replace prio bits, if tcfv_push_prio specified */
                if (p->tcfv_push_prio_exists) {
                        tci &= ~VLAN_PRIO_MASK;
                        tci |= p->tcfv_push_prio << VLAN_PRIO_SHIFT;
                }
                /* put updated tci as hwaccel tag */
                __vlan_hwaccel_put_tag(skb, p->tcfv_push_proto, tci);
                break;
        case TCA_VLAN_ACT_POP_ETH:
                err = skb_eth_pop(skb);
                if (err)
                        goto drop;
                break;
        case TCA_VLAN_ACT_PUSH_ETH:
                err = skb_eth_push(skb, p->tcfv_push_dst, p->tcfv_push_src);
                if (err)
                        goto drop;
                break;
        default:
                BUG();
        }

out:
        if (skb_at_tc_ingress(skb))
                skb_pull_rcsum(skb, skb->mac_len);

        skb_reset_mac_len(skb);
        return p->action;

drop:
        tcf_action_inc_drop_qstats(&v->common);
        return TC_ACT_SHOT;
}

static const struct nla_policy vlan_policy[TCA_VLAN_MAX + 1] = {
        [TCA_VLAN_UNSPEC]                = { .strict_start_type = TCA_VLAN_PUSH_ETH_DST },
        [TCA_VLAN_PARMS]                = { .len = sizeof(struct tc_vlan) },
        [TCA_VLAN_PUSH_VLAN_ID]                = { .type = NLA_U16 },
        [TCA_VLAN_PUSH_VLAN_PROTOCOL]        = { .type = NLA_U16 },
        [TCA_VLAN_PUSH_VLAN_PRIORITY]        = { .type = NLA_U8 },
        [TCA_VLAN_PUSH_ETH_DST]                = NLA_POLICY_ETH_ADDR,
        [TCA_VLAN_PUSH_ETH_SRC]                = NLA_POLICY_ETH_ADDR,
};

static int tcf_vlan_init(struct net *net, struct nlattr *nla,
                         struct nlattr *est, struct tc_action **a,
                         struct tcf_proto *tp, u32 flags,
                         struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, act_vlan_ops.net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct nlattr *tb[TCA_VLAN_MAX + 1];
        struct tcf_chain *goto_ch = NULL;
        bool push_prio_exists = false;
        struct tcf_vlan_params *p;
        struct tc_vlan *parm;
        struct tcf_vlan *v;
        int action;
        u16 push_vid = 0;
        __be16 push_proto = 0;
        u8 push_prio = 0;
        bool exists = false;
        int ret = 0, err;
        u32 index;

        if (!nla)
                return -EINVAL;

        err = nla_parse_nested_deprecated(tb, TCA_VLAN_MAX, nla, vlan_policy,
                                          NULL);
        if (err < 0)
                return err;

        if (!tb[TCA_VLAN_PARMS])
                return -EINVAL;
        parm = nla_data(tb[TCA_VLAN_PARMS]);
        index = parm->index;
        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (err < 0)
                return err;
        exists = err;
        if (exists && bind)
                return ACT_P_BOUND;

        switch (parm->v_action) {
        case TCA_VLAN_ACT_POP:
                break;
        case TCA_VLAN_ACT_PUSH:
        case TCA_VLAN_ACT_MODIFY:
                if (!tb[TCA_VLAN_PUSH_VLAN_ID]) {
                        if (exists)
                                tcf_idr_release(*a, bind);
                        else
                                tcf_idr_cleanup(tn, index);
                        return -EINVAL;
                }
                push_vid = nla_get_u16(tb[TCA_VLAN_PUSH_VLAN_ID]);
                if (push_vid >= VLAN_VID_MASK) {
                        if (exists)
                                tcf_idr_release(*a, bind);
                        else
                                tcf_idr_cleanup(tn, index);
                        return -ERANGE;
                }

                if (tb[TCA_VLAN_PUSH_VLAN_PROTOCOL]) {
                        push_proto = nla_get_be16(tb[TCA_VLAN_PUSH_VLAN_PROTOCOL]);
                        switch (push_proto) {
                        case htons(ETH_P_8021Q):
                        case htons(ETH_P_8021AD):
                                break;
                        default:
                                if (exists)
                                        tcf_idr_release(*a, bind);
                                else
                                        tcf_idr_cleanup(tn, index);
                                return -EPROTONOSUPPORT;
                        }
                } else {
                        push_proto = htons(ETH_P_8021Q);
                }

                push_prio_exists = !!tb[TCA_VLAN_PUSH_VLAN_PRIORITY];
                if (push_prio_exists)
                        push_prio = nla_get_u8(tb[TCA_VLAN_PUSH_VLAN_PRIORITY]);
                break;
        case TCA_VLAN_ACT_POP_ETH:
                break;
        case TCA_VLAN_ACT_PUSH_ETH:
                if (!tb[TCA_VLAN_PUSH_ETH_DST] || !tb[TCA_VLAN_PUSH_ETH_SRC]) {
                        if (exists)
                                tcf_idr_release(*a, bind);
                        else
                                tcf_idr_cleanup(tn, index);
                        return -EINVAL;
                }
                break;
        default:
                if (exists)
                        tcf_idr_release(*a, bind);
                else
                        tcf_idr_cleanup(tn, index);
                return -EINVAL;
        }
        action = parm->v_action;

        if (!exists) {
                ret = tcf_idr_create_from_flags(tn, index, est, a,
                                                &act_vlan_ops, bind, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        return ret;
                }

                ret = ACT_P_CREATED;
        } else if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                tcf_idr_release(*a, bind);
                return -EEXIST;
        }

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0)
                goto release_idr;

        v = to_vlan(*a);

        p = kzalloc_obj(*p);
        if (!p) {
                err = -ENOMEM;
                goto put_chain;
        }

        p->tcfv_action = action;
        p->tcfv_push_vid = push_vid;
        p->tcfv_push_prio = push_prio;
        p->tcfv_push_prio_exists = push_prio_exists || action == TCA_VLAN_ACT_PUSH;
        p->tcfv_push_proto = push_proto;

        if (action == TCA_VLAN_ACT_PUSH_ETH) {
                nla_memcpy(&p->tcfv_push_dst, tb[TCA_VLAN_PUSH_ETH_DST],
                           ETH_ALEN);
                nla_memcpy(&p->tcfv_push_src, tb[TCA_VLAN_PUSH_ETH_SRC],
                           ETH_ALEN);
        }

        p->action = parm->action;
        spin_lock_bh(&v->tcf_lock);
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
        p = rcu_replace_pointer(v->vlan_p, p, lockdep_is_held(&v->tcf_lock));
        spin_unlock_bh(&v->tcf_lock);

        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
        if (p)
                kfree_rcu(p, rcu);

        return ret;
put_chain:
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
release_idr:
        tcf_idr_release(*a, bind);
        return err;
}

static void tcf_vlan_cleanup(struct tc_action *a)
{
        struct tcf_vlan *v = to_vlan(a);
        struct tcf_vlan_params *p;

        p = rcu_dereference_protected(v->vlan_p, 1);
        if (p)
                kfree_rcu(p, rcu);
}

static int tcf_vlan_dump(struct sk_buff *skb, struct tc_action *a,
                         int bind, int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        struct tcf_vlan *v = to_vlan(a);
        struct tcf_vlan_params *p;
        struct tc_vlan opt = {
                .index    = v->tcf_index,
                .refcnt   = refcount_read(&v->tcf_refcnt) - ref,
                .bindcnt  = atomic_read(&v->tcf_bindcnt) - bind,
        };
        struct tcf_t t;

        rcu_read_lock();
        p = rcu_dereference(v->vlan_p);
        opt.action = p->action;
        opt.v_action = p->tcfv_action;
        if (nla_put(skb, TCA_VLAN_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        if ((p->tcfv_action == TCA_VLAN_ACT_PUSH ||
             p->tcfv_action == TCA_VLAN_ACT_MODIFY) &&
            (nla_put_u16(skb, TCA_VLAN_PUSH_VLAN_ID, p->tcfv_push_vid) ||
             nla_put_be16(skb, TCA_VLAN_PUSH_VLAN_PROTOCOL,
                          p->tcfv_push_proto) ||
             (p->tcfv_push_prio_exists &&
              nla_put_u8(skb, TCA_VLAN_PUSH_VLAN_PRIORITY, p->tcfv_push_prio))))
                goto nla_put_failure;

        if (p->tcfv_action == TCA_VLAN_ACT_PUSH_ETH) {
                if (nla_put(skb, TCA_VLAN_PUSH_ETH_DST, ETH_ALEN,
                            p->tcfv_push_dst))
                        goto nla_put_failure;
                if (nla_put(skb, TCA_VLAN_PUSH_ETH_SRC, ETH_ALEN,
                            p->tcfv_push_src))
                        goto nla_put_failure;
        }

        tcf_tm_dump(&t, &v->tcf_tm);
        if (nla_put_64bit(skb, TCA_VLAN_TM, sizeof(t), &t, TCA_VLAN_PAD))
                goto nla_put_failure;
        rcu_read_unlock();

        return skb->len;

nla_put_failure:
        rcu_read_unlock();
        nlmsg_trim(skb, b);
        return -1;
}

static void tcf_vlan_stats_update(struct tc_action *a, u64 bytes, u64 packets,
                                  u64 drops, u64 lastuse, bool hw)
{
        struct tcf_vlan *v = to_vlan(a);
        struct tcf_t *tm = &v->tcf_tm;

        tcf_action_update_stats(a, bytes, packets, drops, hw);
        tm->lastuse = max_t(u64, tm->lastuse, lastuse);
}

static size_t tcf_vlan_get_fill_size(const struct tc_action *act)
{
        return nla_total_size(sizeof(struct tc_vlan))
                + nla_total_size(sizeof(u16)) /* TCA_VLAN_PUSH_VLAN_ID */
                + nla_total_size(sizeof(u16)) /* TCA_VLAN_PUSH_VLAN_PROTOCOL */
                + nla_total_size(sizeof(u8)); /* TCA_VLAN_PUSH_VLAN_PRIORITY */
}

static int tcf_vlan_offload_act_setup(struct tc_action *act, void *entry_data,
                                      u32 *index_inc, bool bind,
                                      struct netlink_ext_ack *extack)
{
        if (bind) {
                struct flow_action_entry *entry = entry_data;

                switch (tcf_vlan_action(act)) {
                case TCA_VLAN_ACT_PUSH:
                        entry->id = FLOW_ACTION_VLAN_PUSH;
                        entry->vlan.vid = tcf_vlan_push_vid(act);
                        entry->vlan.proto = tcf_vlan_push_proto(act);
                        entry->vlan.prio = tcf_vlan_push_prio(act);
                        break;
                case TCA_VLAN_ACT_POP:
                        entry->id = FLOW_ACTION_VLAN_POP;
                        break;
                case TCA_VLAN_ACT_MODIFY:
                        entry->id = FLOW_ACTION_VLAN_MANGLE;
                        entry->vlan.vid = tcf_vlan_push_vid(act);
                        entry->vlan.proto = tcf_vlan_push_proto(act);
                        entry->vlan.prio = tcf_vlan_push_prio(act);
                        break;
                case TCA_VLAN_ACT_POP_ETH:
                        entry->id = FLOW_ACTION_VLAN_POP_ETH;
                        break;
                case TCA_VLAN_ACT_PUSH_ETH:
                        entry->id = FLOW_ACTION_VLAN_PUSH_ETH;
                        tcf_vlan_push_eth(entry->vlan_push_eth.src, entry->vlan_push_eth.dst, act);
                        break;
                default:
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported vlan action mode offload");
                        return -EOPNOTSUPP;
                }
                *index_inc = 1;
        } else {
                struct flow_offload_action *fl_action = entry_data;

                switch (tcf_vlan_action(act)) {
                case TCA_VLAN_ACT_PUSH:
                        fl_action->id = FLOW_ACTION_VLAN_PUSH;
                        break;
                case TCA_VLAN_ACT_POP:
                        fl_action->id = FLOW_ACTION_VLAN_POP;
                        break;
                case TCA_VLAN_ACT_MODIFY:
                        fl_action->id = FLOW_ACTION_VLAN_MANGLE;
                        break;
                case TCA_VLAN_ACT_POP_ETH:
                        fl_action->id = FLOW_ACTION_VLAN_POP_ETH;
                        break;
                case TCA_VLAN_ACT_PUSH_ETH:
                        fl_action->id = FLOW_ACTION_VLAN_PUSH_ETH;
                        break;
                default:
                        return -EOPNOTSUPP;
                }
        }

        return 0;
}

static struct tc_action_ops act_vlan_ops = {
        .kind                =        "vlan",
        .id                =        TCA_ID_VLAN,
        .owner                =        THIS_MODULE,
        .act                =        tcf_vlan_act,
        .dump                =        tcf_vlan_dump,
        .init                =        tcf_vlan_init,
        .cleanup        =        tcf_vlan_cleanup,
        .stats_update        =        tcf_vlan_stats_update,
        .get_fill_size        =        tcf_vlan_get_fill_size,
        .offload_act_setup =        tcf_vlan_offload_act_setup,
        .size                =        sizeof(struct tcf_vlan),
};
MODULE_ALIAS_NET_ACT("vlan");

static __net_init int vlan_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, act_vlan_ops.net_id);

        return tc_action_net_init(net, tn, &act_vlan_ops);
}

static void __net_exit vlan_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, act_vlan_ops.net_id);
}

static struct pernet_operations vlan_net_ops = {
        .init = vlan_init_net,
        .exit_batch = vlan_exit_net,
        .id   = &act_vlan_ops.net_id,
        .size = sizeof(struct tc_action_net),
};

static int __init vlan_init_module(void)
{
        return tcf_register_action(&act_vlan_ops, &vlan_net_ops);
}

static void __exit vlan_cleanup_module(void)
{
        tcf_unregister_action(&act_vlan_ops, &vlan_net_ops);
}

module_init(vlan_init_module);
module_exit(vlan_cleanup_module);

MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
MODULE_DESCRIPTION("vlan manipulation actions");
MODULE_LICENSE("GPL v2");




















































































 1171 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Prevent the compiler from merging or refetching reads or writes. The
 * compiler is also forbidden from reordering successive instances of
 * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some
 * particular ordering. One way to make the compiler aware of ordering is to
 * put the two invocations of READ_ONCE or WRITE_ONCE in different C
 * statements.
 *
 * These two macros will also work on aggregate data types like structs or
 * unions.
 *
 * Their two major use cases are: (1) Mediating communication between
 * process-level code and irq/NMI handlers, all running on the same CPU,
 * and (2) Ensuring that the compiler does not fold, spindle, or otherwise
 * mutilate accesses that either do not require ordering or that interact
 * with an explicit memory barrier or atomic instruction that provides the
 * required ordering.
 */
#ifndef __ASM_GENERIC_RWONCE_H
#define __ASM_GENERIC_RWONCE_H

#ifndef __ASSEMBLY__

#include <linux/compiler_types.h>
#include <linux/kasan-checks.h>
#include <linux/kcsan-checks.h>

/*
 * Yes, this permits 64-bit accesses on 32-bit architectures. These will
 * actually be atomic in some cases (namely Armv7 + LPAE), but for others we
 * rely on the access being split into 2x32-bit accesses for a 32-bit quantity
 * (e.g. a virtual address) and a strong prevailing wind.
 */
#define compiletime_assert_rwonce_type(t)                                        \
        compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long),        \
                "Unsupported access size for {READ,WRITE}_ONCE().")

/*
 * Use __READ_ONCE() instead of READ_ONCE() if you do not require any
 * atomicity. Note that this may result in tears!
 */
#ifndef __READ_ONCE
#define __READ_ONCE(x)        (*(const volatile __unqual_scalar_typeof(x) *)&(x))
#endif

#define READ_ONCE(x)                                                        \
({                                                                        \
        compiletime_assert_rwonce_type(x);                                \
        __READ_ONCE(x);                                                        \
})

#define __WRITE_ONCE(x, val)                                                \
do {                                                                        \
        *(volatile typeof(x) *)&(x) = (val);                                \
} while (0)

#define WRITE_ONCE(x, val)                                                \
do {                                                                        \
        compiletime_assert_rwonce_type(x);                                \
        __WRITE_ONCE(x, val);                                                \
} while (0)

static __no_sanitize_or_inline
unsigned long __read_once_word_nocheck(const void *addr)
{
        return __READ_ONCE(*(unsigned long *)addr);
}

/*
 * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a
 * word from memory atomically but without telling KASAN/KCSAN. This is
 * usually used by unwinding code when walking the stack of a running process.
 */
#define READ_ONCE_NOCHECK(x)                                                \
({                                                                        \
        compiletime_assert(sizeof(x) == sizeof(unsigned long),                \
                "Unsupported access size for READ_ONCE_NOCHECK().");        \
        (typeof(x))__read_once_word_nocheck(&(x));                        \
})

static __no_sanitize_or_inline
unsigned long read_word_at_a_time(const void *addr)
{
        /* open-coded instrument_read(addr, 1) */
        kasan_check_read(addr, 1);
        kcsan_check_read(addr, 1);

        /*
         * This load can race with concurrent stores to out-of-bounds memory,
         * but READ_ONCE() can't be used because it requires higher alignment
         * than plain loads in arm64 builds with LTO.
         */
        return *(unsigned long *)addr;
}

#endif /* __ASSEMBLY__ */
#endif        /* __ASM_GENERIC_RWONCE_H */
























































   14 
   14 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
// SPDX-License-Identifier: GPL-2.0-only
/*
 *      crc16.c
 */

#include <linux/crc16.h>
#include <linux/export.h>
#include <linux/module.h>
#include <linux/types.h>

/** CRC table for the CRC-16. The poly is 0x8005 (x^16 + x^15 + x^2 + 1) */
static const u16 crc16_table[256] = {
        0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241,
        0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440,
        0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40,
        0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841,
        0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40,
        0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41,
        0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641,
        0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040,
        0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240,
        0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441,
        0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41,
        0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840,
        0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41,
        0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40,
        0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640,
        0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041,
        0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240,
        0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441,
        0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41,
        0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840,
        0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41,
        0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40,
        0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640,
        0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041,
        0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241,
        0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440,
        0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40,
        0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841,
        0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40,
        0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41,
        0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641,
        0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040
};

/**
 * crc16 - compute the CRC-16 for the data buffer
 * @crc:        previous CRC value
 * @p:                data pointer
 * @len:        number of bytes in the buffer
 *
 * Returns the updated CRC value.
 */
u16 crc16(u16 crc, const u8 *p, size_t len)
{
        while (len--)
                crc = (crc >> 8) ^ crc16_table[(crc & 0xff) ^ *p++];
        return crc;
}
EXPORT_SYMBOL(crc16);

MODULE_DESCRIPTION("CRC16 calculations");
MODULE_LICENSE("GPL");





















































































   64 


 1082 
  142 

























  284 


 3724 













  215 





  216 
  216 






























  935 




















































 3818 



































  932 



  936 

  723 
















 3816 



 3826 
 3511 
 3323 
  470 



































































































































































































































































































  936 


  935 








  935 
  933 

























































































































 3814 








 3816 






 3640 
  215 






 3818 

 3808 

 3812 
















  862 




















  225 


  659 
  488 






  864 
















  124 










  123 






















  124 




  124 










































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
// SPDX-License-Identifier: GPL-2.0
/*
 * security/tomoyo/file.c
 *
 * Copyright (C) 2005-2011  NTT DATA CORPORATION
 */

#include "common.h"
#include <linux/slab.h>

/*
 * Mapping table from "enum tomoyo_path_acl_index" to "enum tomoyo_mac_index".
 */
static const u8 tomoyo_p2mac[TOMOYO_MAX_PATH_OPERATION] = {
        [TOMOYO_TYPE_EXECUTE]    = TOMOYO_MAC_FILE_EXECUTE,
        [TOMOYO_TYPE_READ]       = TOMOYO_MAC_FILE_OPEN,
        [TOMOYO_TYPE_WRITE]      = TOMOYO_MAC_FILE_OPEN,
        [TOMOYO_TYPE_APPEND]     = TOMOYO_MAC_FILE_OPEN,
        [TOMOYO_TYPE_UNLINK]     = TOMOYO_MAC_FILE_UNLINK,
        [TOMOYO_TYPE_GETATTR]    = TOMOYO_MAC_FILE_GETATTR,
        [TOMOYO_TYPE_RMDIR]      = TOMOYO_MAC_FILE_RMDIR,
        [TOMOYO_TYPE_TRUNCATE]   = TOMOYO_MAC_FILE_TRUNCATE,
        [TOMOYO_TYPE_SYMLINK]    = TOMOYO_MAC_FILE_SYMLINK,
        [TOMOYO_TYPE_CHROOT]     = TOMOYO_MAC_FILE_CHROOT,
        [TOMOYO_TYPE_UMOUNT]     = TOMOYO_MAC_FILE_UMOUNT,
};

/*
 * Mapping table from "enum tomoyo_mkdev_acl_index" to "enum tomoyo_mac_index".
 */
const u8 tomoyo_pnnn2mac[TOMOYO_MAX_MKDEV_OPERATION] = {
        [TOMOYO_TYPE_MKBLOCK] = TOMOYO_MAC_FILE_MKBLOCK,
        [TOMOYO_TYPE_MKCHAR]  = TOMOYO_MAC_FILE_MKCHAR,
};

/*
 * Mapping table from "enum tomoyo_path2_acl_index" to "enum tomoyo_mac_index".
 */
const u8 tomoyo_pp2mac[TOMOYO_MAX_PATH2_OPERATION] = {
        [TOMOYO_TYPE_LINK]       = TOMOYO_MAC_FILE_LINK,
        [TOMOYO_TYPE_RENAME]     = TOMOYO_MAC_FILE_RENAME,
        [TOMOYO_TYPE_PIVOT_ROOT] = TOMOYO_MAC_FILE_PIVOT_ROOT,
};

/*
 * Mapping table from "enum tomoyo_path_number_acl_index" to
 * "enum tomoyo_mac_index".
 */
const u8 tomoyo_pn2mac[TOMOYO_MAX_PATH_NUMBER_OPERATION] = {
        [TOMOYO_TYPE_CREATE] = TOMOYO_MAC_FILE_CREATE,
        [TOMOYO_TYPE_MKDIR]  = TOMOYO_MAC_FILE_MKDIR,
        [TOMOYO_TYPE_MKFIFO] = TOMOYO_MAC_FILE_MKFIFO,
        [TOMOYO_TYPE_MKSOCK] = TOMOYO_MAC_FILE_MKSOCK,
        [TOMOYO_TYPE_IOCTL]  = TOMOYO_MAC_FILE_IOCTL,
        [TOMOYO_TYPE_CHMOD]  = TOMOYO_MAC_FILE_CHMOD,
        [TOMOYO_TYPE_CHOWN]  = TOMOYO_MAC_FILE_CHOWN,
        [TOMOYO_TYPE_CHGRP]  = TOMOYO_MAC_FILE_CHGRP,
};

/**
 * tomoyo_put_name_union - Drop reference on "struct tomoyo_name_union".
 *
 * @ptr: Pointer to "struct tomoyo_name_union".
 *
 * Returns nothing.
 */
void tomoyo_put_name_union(struct tomoyo_name_union *ptr)
{
        tomoyo_put_group(ptr->group);
        tomoyo_put_name(ptr->filename);
}

/**
 * tomoyo_compare_name_union - Check whether a name matches "struct tomoyo_name_union" or not.
 *
 * @name: Pointer to "struct tomoyo_path_info".
 * @ptr:  Pointer to "struct tomoyo_name_union".
 *
 * Returns "struct tomoyo_path_info" if @name matches @ptr, NULL otherwise.
 */
const struct tomoyo_path_info *
tomoyo_compare_name_union(const struct tomoyo_path_info *name,
                          const struct tomoyo_name_union *ptr)
{
        if (ptr->group)
                return tomoyo_path_matches_group(name, ptr->group);
        if (tomoyo_path_matches_pattern(name, ptr->filename))
                return ptr->filename;
        return NULL;
}

/**
 * tomoyo_put_number_union - Drop reference on "struct tomoyo_number_union".
 *
 * @ptr: Pointer to "struct tomoyo_number_union".
 *
 * Returns nothing.
 */
void tomoyo_put_number_union(struct tomoyo_number_union *ptr)
{
        tomoyo_put_group(ptr->group);
}

/**
 * tomoyo_compare_number_union - Check whether a value matches "struct tomoyo_number_union" or not.
 *
 * @value: Number to check.
 * @ptr:   Pointer to "struct tomoyo_number_union".
 *
 * Returns true if @value matches @ptr, false otherwise.
 */
bool tomoyo_compare_number_union(const unsigned long value,
                                 const struct tomoyo_number_union *ptr)
{
        if (ptr->group)
                return tomoyo_number_matches_group(value, value, ptr->group);
        return value >= ptr->values[0] && value <= ptr->values[1];
}

/**
 * tomoyo_add_slash - Add trailing '/' if needed.
 *
 * @buf: Pointer to "struct tomoyo_path_info".
 *
 * Returns nothing.
 *
 * @buf must be generated by tomoyo_encode() because this function does not
 * allocate memory for adding '/'.
 */
static void tomoyo_add_slash(struct tomoyo_path_info *buf)
{
        if (buf->is_dir)
                return;
        /*
         * This is OK because tomoyo_encode() reserves space for appending "/".
         */
        strcat((char *) buf->name, "/");
        tomoyo_fill_path_info(buf);
}

/**
 * tomoyo_get_realpath - Get realpath.
 *
 * @buf:  Pointer to "struct tomoyo_path_info".
 * @path: Pointer to "struct path".
 *
 * Returns true on success, false otherwise.
 */
static bool tomoyo_get_realpath(struct tomoyo_path_info *buf, const struct path *path)
{
        buf->name = tomoyo_realpath_from_path(path);
        if (buf->name) {
                tomoyo_fill_path_info(buf);
                return true;
        }
        return false;
}

/**
 * tomoyo_audit_path_log - Audit path request log.
 *
 * @r: Pointer to "struct tomoyo_request_info".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_audit_path_log(struct tomoyo_request_info *r)
        __must_hold_shared(&tomoyo_ss)
{
        return tomoyo_supervisor(r, "file %s %s\n", tomoyo_path_keyword
                                 [r->param.path.operation],
                                 r->param.path.filename->name);
}

/**
 * tomoyo_audit_path2_log - Audit path/path request log.
 *
 * @r: Pointer to "struct tomoyo_request_info".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_audit_path2_log(struct tomoyo_request_info *r)
        __must_hold_shared(&tomoyo_ss)
{
        return tomoyo_supervisor(r, "file %s %s %s\n", tomoyo_mac_keywords
                                 [tomoyo_pp2mac[r->param.path2.operation]],
                                 r->param.path2.filename1->name,
                                 r->param.path2.filename2->name);
}

/**
 * tomoyo_audit_mkdev_log - Audit path/number/number/number request log.
 *
 * @r: Pointer to "struct tomoyo_request_info".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_audit_mkdev_log(struct tomoyo_request_info *r)
        __must_hold_shared(&tomoyo_ss)
{
        return tomoyo_supervisor(r, "file %s %s 0%o %u %u\n",
                                 tomoyo_mac_keywords
                                 [tomoyo_pnnn2mac[r->param.mkdev.operation]],
                                 r->param.mkdev.filename->name,
                                 r->param.mkdev.mode, r->param.mkdev.major,
                                 r->param.mkdev.minor);
}

/**
 * tomoyo_audit_path_number_log - Audit path/number request log.
 *
 * @r: Pointer to "struct tomoyo_request_info".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_audit_path_number_log(struct tomoyo_request_info *r)
        __must_hold_shared(&tomoyo_ss)
{
        const u8 type = r->param.path_number.operation;
        u8 radix;
        char buffer[64];

        switch (type) {
        case TOMOYO_TYPE_CREATE:
        case TOMOYO_TYPE_MKDIR:
        case TOMOYO_TYPE_MKFIFO:
        case TOMOYO_TYPE_MKSOCK:
        case TOMOYO_TYPE_CHMOD:
                radix = TOMOYO_VALUE_TYPE_OCTAL;
                break;
        case TOMOYO_TYPE_IOCTL:
                radix = TOMOYO_VALUE_TYPE_HEXADECIMAL;
                break;
        default:
                radix = TOMOYO_VALUE_TYPE_DECIMAL;
                break;
        }
        tomoyo_print_ulong(buffer, sizeof(buffer), r->param.path_number.number,
                           radix);
        return tomoyo_supervisor(r, "file %s %s %s\n", tomoyo_mac_keywords
                                 [tomoyo_pn2mac[type]],
                                 r->param.path_number.filename->name, buffer);
}

/**
 * tomoyo_check_path_acl - Check permission for path operation.
 *
 * @r:   Pointer to "struct tomoyo_request_info".
 * @ptr: Pointer to "struct tomoyo_acl_info".
 *
 * Returns true if granted, false otherwise.
 *
 * To be able to use wildcard for domain transition, this function sets
 * matching entry on success. Since the caller holds tomoyo_read_lock(),
 * it is safe to set matching entry.
 */
static bool tomoyo_check_path_acl(struct tomoyo_request_info *r,
                                  const struct tomoyo_acl_info *ptr)
{
        const struct tomoyo_path_acl *acl = container_of(ptr, typeof(*acl),
                                                         head);

        if (acl->perm & (1 << r->param.path.operation)) {
                r->param.path.matched_path =
                        tomoyo_compare_name_union(r->param.path.filename,
                                                  &acl->name);
                return r->param.path.matched_path != NULL;
        }
        return false;
}

/**
 * tomoyo_check_path_number_acl - Check permission for path number operation.
 *
 * @r:   Pointer to "struct tomoyo_request_info".
 * @ptr: Pointer to "struct tomoyo_acl_info".
 *
 * Returns true if granted, false otherwise.
 */
static bool tomoyo_check_path_number_acl(struct tomoyo_request_info *r,
                                         const struct tomoyo_acl_info *ptr)
{
        const struct tomoyo_path_number_acl *acl =
                container_of(ptr, typeof(*acl), head);

        return (acl->perm & (1 << r->param.path_number.operation)) &&
                tomoyo_compare_number_union(r->param.path_number.number,
                                            &acl->number) &&
                tomoyo_compare_name_union(r->param.path_number.filename,
                                          &acl->name);
}

/**
 * tomoyo_check_path2_acl - Check permission for path path operation.
 *
 * @r:   Pointer to "struct tomoyo_request_info".
 * @ptr: Pointer to "struct tomoyo_acl_info".
 *
 * Returns true if granted, false otherwise.
 */
static bool tomoyo_check_path2_acl(struct tomoyo_request_info *r,
                                   const struct tomoyo_acl_info *ptr)
{
        const struct tomoyo_path2_acl *acl =
                container_of(ptr, typeof(*acl), head);

        return (acl->perm & (1 << r->param.path2.operation)) &&
                tomoyo_compare_name_union(r->param.path2.filename1, &acl->name1)
                && tomoyo_compare_name_union(r->param.path2.filename2,
                                             &acl->name2);
}

/**
 * tomoyo_check_mkdev_acl - Check permission for path number number number operation.
 *
 * @r:   Pointer to "struct tomoyo_request_info".
 * @ptr: Pointer to "struct tomoyo_acl_info".
 *
 * Returns true if granted, false otherwise.
 */
static bool tomoyo_check_mkdev_acl(struct tomoyo_request_info *r,
                                   const struct tomoyo_acl_info *ptr)
{
        const struct tomoyo_mkdev_acl *acl =
                container_of(ptr, typeof(*acl), head);

        return (acl->perm & (1 << r->param.mkdev.operation)) &&
                tomoyo_compare_number_union(r->param.mkdev.mode,
                                            &acl->mode) &&
                tomoyo_compare_number_union(r->param.mkdev.major,
                                            &acl->major) &&
                tomoyo_compare_number_union(r->param.mkdev.minor,
                                            &acl->minor) &&
                tomoyo_compare_name_union(r->param.mkdev.filename,
                                          &acl->name);
}

/**
 * tomoyo_same_path_acl - Check for duplicated "struct tomoyo_path_acl" entry.
 *
 * @a: Pointer to "struct tomoyo_acl_info".
 * @b: Pointer to "struct tomoyo_acl_info".
 *
 * Returns true if @a == @b except permission bits, false otherwise.
 */
static bool tomoyo_same_path_acl(const struct tomoyo_acl_info *a,
                                 const struct tomoyo_acl_info *b)
{
        const struct tomoyo_path_acl *p1 = container_of(a, typeof(*p1), head);
        const struct tomoyo_path_acl *p2 = container_of(b, typeof(*p2), head);

        return tomoyo_same_name_union(&p1->name, &p2->name);
}

/**
 * tomoyo_merge_path_acl - Merge duplicated "struct tomoyo_path_acl" entry.
 *
 * @a:         Pointer to "struct tomoyo_acl_info".
 * @b:         Pointer to "struct tomoyo_acl_info".
 * @is_delete: True for @a &= ~@b, false for @a |= @b.
 *
 * Returns true if @a is empty, false otherwise.
 */
static bool tomoyo_merge_path_acl(struct tomoyo_acl_info *a,
                                  struct tomoyo_acl_info *b,
                                  const bool is_delete)
{
        u16 * const a_perm = &container_of(a, struct tomoyo_path_acl, head)
                ->perm;
        u16 perm = READ_ONCE(*a_perm);
        const u16 b_perm = container_of(b, struct tomoyo_path_acl, head)->perm;

        if (is_delete)
                perm &= ~b_perm;
        else
                perm |= b_perm;
        WRITE_ONCE(*a_perm, perm);
        return !perm;
}

/**
 * tomoyo_update_path_acl - Update "struct tomoyo_path_acl" list.
 *
 * @perm:  Permission.
 * @param: Pointer to "struct tomoyo_acl_param".
 *
 * Returns 0 on success, negative value otherwise.
 *
 * Caller holds tomoyo_read_lock().
 */
static int tomoyo_update_path_acl(const u16 perm,
                                  struct tomoyo_acl_param *param)
{
        struct tomoyo_path_acl e = {
                .head.type = TOMOYO_TYPE_PATH_ACL,
                .perm = perm
        };
        int error;

        if (!tomoyo_parse_name_union(param, &e.name))
                error = -EINVAL;
        else
                error = tomoyo_update_domain(&e.head, sizeof(e), param,
                                             tomoyo_same_path_acl,
                                             tomoyo_merge_path_acl);
        tomoyo_put_name_union(&e.name);
        return error;
}

/**
 * tomoyo_same_mkdev_acl - Check for duplicated "struct tomoyo_mkdev_acl" entry.
 *
 * @a: Pointer to "struct tomoyo_acl_info".
 * @b: Pointer to "struct tomoyo_acl_info".
 *
 * Returns true if @a == @b except permission bits, false otherwise.
 */
static bool tomoyo_same_mkdev_acl(const struct tomoyo_acl_info *a,
                                         const struct tomoyo_acl_info *b)
{
        const struct tomoyo_mkdev_acl *p1 = container_of(a, typeof(*p1), head);
        const struct tomoyo_mkdev_acl *p2 = container_of(b, typeof(*p2), head);

        return tomoyo_same_name_union(&p1->name, &p2->name) &&
                tomoyo_same_number_union(&p1->mode, &p2->mode) &&
                tomoyo_same_number_union(&p1->major, &p2->major) &&
                tomoyo_same_number_union(&p1->minor, &p2->minor);
}

/**
 * tomoyo_merge_mkdev_acl - Merge duplicated "struct tomoyo_mkdev_acl" entry.
 *
 * @a:         Pointer to "struct tomoyo_acl_info".
 * @b:         Pointer to "struct tomoyo_acl_info".
 * @is_delete: True for @a &= ~@b, false for @a |= @b.
 *
 * Returns true if @a is empty, false otherwise.
 */
static bool tomoyo_merge_mkdev_acl(struct tomoyo_acl_info *a,
                                   struct tomoyo_acl_info *b,
                                   const bool is_delete)
{
        u8 *const a_perm = &container_of(a, struct tomoyo_mkdev_acl,
                                         head)->perm;
        u8 perm = READ_ONCE(*a_perm);
        const u8 b_perm = container_of(b, struct tomoyo_mkdev_acl, head)
                ->perm;

        if (is_delete)
                perm &= ~b_perm;
        else
                perm |= b_perm;
        WRITE_ONCE(*a_perm, perm);
        return !perm;
}

/**
 * tomoyo_update_mkdev_acl - Update "struct tomoyo_mkdev_acl" list.
 *
 * @perm:  Permission.
 * @param: Pointer to "struct tomoyo_acl_param".
 *
 * Returns 0 on success, negative value otherwise.
 *
 * Caller holds tomoyo_read_lock().
 */
static int tomoyo_update_mkdev_acl(const u8 perm,
                                   struct tomoyo_acl_param *param)
{
        struct tomoyo_mkdev_acl e = {
                .head.type = TOMOYO_TYPE_MKDEV_ACL,
                .perm = perm
        };
        int error;

        if (!tomoyo_parse_name_union(param, &e.name) ||
            !tomoyo_parse_number_union(param, &e.mode) ||
            !tomoyo_parse_number_union(param, &e.major) ||
            !tomoyo_parse_number_union(param, &e.minor))
                error = -EINVAL;
        else
                error = tomoyo_update_domain(&e.head, sizeof(e), param,
                                             tomoyo_same_mkdev_acl,
                                             tomoyo_merge_mkdev_acl);
        tomoyo_put_name_union(&e.name);
        tomoyo_put_number_union(&e.mode);
        tomoyo_put_number_union(&e.major);
        tomoyo_put_number_union(&e.minor);
        return error;
}

/**
 * tomoyo_same_path2_acl - Check for duplicated "struct tomoyo_path2_acl" entry.
 *
 * @a: Pointer to "struct tomoyo_acl_info".
 * @b: Pointer to "struct tomoyo_acl_info".
 *
 * Returns true if @a == @b except permission bits, false otherwise.
 */
static bool tomoyo_same_path2_acl(const struct tomoyo_acl_info *a,
                                  const struct tomoyo_acl_info *b)
{
        const struct tomoyo_path2_acl *p1 = container_of(a, typeof(*p1), head);
        const struct tomoyo_path2_acl *p2 = container_of(b, typeof(*p2), head);

        return tomoyo_same_name_union(&p1->name1, &p2->name1) &&
                tomoyo_same_name_union(&p1->name2, &p2->name2);
}

/**
 * tomoyo_merge_path2_acl - Merge duplicated "struct tomoyo_path2_acl" entry.
 *
 * @a:         Pointer to "struct tomoyo_acl_info".
 * @b:         Pointer to "struct tomoyo_acl_info".
 * @is_delete: True for @a &= ~@b, false for @a |= @b.
 *
 * Returns true if @a is empty, false otherwise.
 */
static bool tomoyo_merge_path2_acl(struct tomoyo_acl_info *a,
                                   struct tomoyo_acl_info *b,
                                   const bool is_delete)
{
        u8 * const a_perm = &container_of(a, struct tomoyo_path2_acl, head)
                ->perm;
        u8 perm = READ_ONCE(*a_perm);
        const u8 b_perm = container_of(b, struct tomoyo_path2_acl, head)->perm;

        if (is_delete)
                perm &= ~b_perm;
        else
                perm |= b_perm;
        WRITE_ONCE(*a_perm, perm);
        return !perm;
}

/**
 * tomoyo_update_path2_acl - Update "struct tomoyo_path2_acl" list.
 *
 * @perm:  Permission.
 * @param: Pointer to "struct tomoyo_acl_param".
 *
 * Returns 0 on success, negative value otherwise.
 *
 * Caller holds tomoyo_read_lock().
 */
static int tomoyo_update_path2_acl(const u8 perm,
                                   struct tomoyo_acl_param *param)
{
        struct tomoyo_path2_acl e = {
                .head.type = TOMOYO_TYPE_PATH2_ACL,
                .perm = perm
        };
        int error;

        if (!tomoyo_parse_name_union(param, &e.name1) ||
            !tomoyo_parse_name_union(param, &e.name2))
                error = -EINVAL;
        else
                error = tomoyo_update_domain(&e.head, sizeof(e), param,
                                             tomoyo_same_path2_acl,
                                             tomoyo_merge_path2_acl);
        tomoyo_put_name_union(&e.name1);
        tomoyo_put_name_union(&e.name2);
        return error;
}

/**
 * tomoyo_path_permission - Check permission for single path operation.
 *
 * @r:         Pointer to "struct tomoyo_request_info".
 * @operation: Type of operation.
 * @filename:  Filename to check.
 *
 * Returns 0 on success, negative value otherwise.
 *
 * Caller holds tomoyo_read_lock().
 */
static int tomoyo_path_permission(struct tomoyo_request_info *r, u8 operation,
                                  const struct tomoyo_path_info *filename)
        __must_hold_shared(&tomoyo_ss)
{
        int error;

        r->type = tomoyo_p2mac[operation];
        r->mode = tomoyo_get_mode(r->domain->ns, r->profile, r->type);
        if (r->mode == TOMOYO_CONFIG_DISABLED)
                return 0;
        r->param_type = TOMOYO_TYPE_PATH_ACL;
        r->param.path.filename = filename;
        r->param.path.operation = operation;
        do {
                tomoyo_check_acl(r, tomoyo_check_path_acl);
                error = tomoyo_audit_path_log(r);
        } while (error == TOMOYO_RETRY_REQUEST);
        return error;
}

/**
 * tomoyo_execute_permission - Check permission for execute operation.
 *
 * @r:         Pointer to "struct tomoyo_request_info".
 * @filename:  Filename to check.
 *
 * Returns 0 on success, negative value otherwise.
 *
 * Caller holds tomoyo_read_lock().
 */
int tomoyo_execute_permission(struct tomoyo_request_info *r,
                              const struct tomoyo_path_info *filename)
{
        /*
         * Unlike other permission checks, this check is done regardless of
         * profile mode settings in order to check for domain transition
         * preference.
         */
        r->type = TOMOYO_MAC_FILE_EXECUTE;
        r->mode = tomoyo_get_mode(r->domain->ns, r->profile, r->type);
        r->param_type = TOMOYO_TYPE_PATH_ACL;
        r->param.path.filename = filename;
        r->param.path.operation = TOMOYO_TYPE_EXECUTE;
        tomoyo_check_acl(r, tomoyo_check_path_acl);
        r->ee->transition = r->matched_acl && r->matched_acl->cond ?
                r->matched_acl->cond->transit : NULL;
        if (r->mode != TOMOYO_CONFIG_DISABLED)
                return tomoyo_audit_path_log(r);
        return 0;
}

/**
 * tomoyo_same_path_number_acl - Check for duplicated "struct tomoyo_path_number_acl" entry.
 *
 * @a: Pointer to "struct tomoyo_acl_info".
 * @b: Pointer to "struct tomoyo_acl_info".
 *
 * Returns true if @a == @b except permission bits, false otherwise.
 */
static bool tomoyo_same_path_number_acl(const struct tomoyo_acl_info *a,
                                        const struct tomoyo_acl_info *b)
{
        const struct tomoyo_path_number_acl *p1 = container_of(a, typeof(*p1),
                                                               head);
        const struct tomoyo_path_number_acl *p2 = container_of(b, typeof(*p2),
                                                               head);

        return tomoyo_same_name_union(&p1->name, &p2->name) &&
                tomoyo_same_number_union(&p1->number, &p2->number);
}

/**
 * tomoyo_merge_path_number_acl - Merge duplicated "struct tomoyo_path_number_acl" entry.
 *
 * @a:         Pointer to "struct tomoyo_acl_info".
 * @b:         Pointer to "struct tomoyo_acl_info".
 * @is_delete: True for @a &= ~@b, false for @a |= @b.
 *
 * Returns true if @a is empty, false otherwise.
 */
static bool tomoyo_merge_path_number_acl(struct tomoyo_acl_info *a,
                                         struct tomoyo_acl_info *b,
                                         const bool is_delete)
{
        u8 * const a_perm = &container_of(a, struct tomoyo_path_number_acl,
                                          head)->perm;
        u8 perm = READ_ONCE(*a_perm);
        const u8 b_perm = container_of(b, struct tomoyo_path_number_acl, head)
                ->perm;

        if (is_delete)
                perm &= ~b_perm;
        else
                perm |= b_perm;
        WRITE_ONCE(*a_perm, perm);
        return !perm;
}

/**
 * tomoyo_update_path_number_acl - Update ioctl/chmod/chown/chgrp ACL.
 *
 * @perm:  Permission.
 * @param: Pointer to "struct tomoyo_acl_param".
 *
 * Returns 0 on success, negative value otherwise.
 */
static int tomoyo_update_path_number_acl(const u8 perm,
                                         struct tomoyo_acl_param *param)
{
        struct tomoyo_path_number_acl e = {
                .head.type = TOMOYO_TYPE_PATH_NUMBER_ACL,
                .perm = perm
        };
        int error;

        if (!tomoyo_parse_name_union(param, &e.name) ||
            !tomoyo_parse_number_union(param, &e.number))
                error = -EINVAL;
        else
                error = tomoyo_update_domain(&e.head, sizeof(e), param,
                                             tomoyo_same_path_number_acl,
                                             tomoyo_merge_path_number_acl);
        tomoyo_put_name_union(&e.name);
        tomoyo_put_number_union(&e.number);
        return error;
}

/**
 * tomoyo_path_number_perm - Check permission for "create", "mkdir", "mkfifo", "mksock", "ioctl", "chmod", "chown", "chgrp".
 *
 * @type:   Type of operation.
 * @path:   Pointer to "struct path".
 * @number: Number.
 *
 * Returns 0 on success, negative value otherwise.
 */
int tomoyo_path_number_perm(const u8 type, const struct path *path,
                            unsigned long number)
{
        struct tomoyo_request_info r;
        struct tomoyo_obj_info obj = {
                .path1 = { .mnt = path->mnt, .dentry = path->dentry },
        };
        int error = -ENOMEM;
        struct tomoyo_path_info buf;
        int idx;

        if (tomoyo_init_request_info(&r, NULL, tomoyo_pn2mac[type])
            == TOMOYO_CONFIG_DISABLED)
                return 0;
        idx = tomoyo_read_lock();
        if (!tomoyo_get_realpath(&buf, path))
                goto out;
        r.obj = &obj;
        if (type == TOMOYO_TYPE_MKDIR)
                tomoyo_add_slash(&buf);
        r.param_type = TOMOYO_TYPE_PATH_NUMBER_ACL;
        r.param.path_number.operation = type;
        r.param.path_number.filename = &buf;
        r.param.path_number.number = number;
        do {
                tomoyo_check_acl(&r, tomoyo_check_path_number_acl);
                error = tomoyo_audit_path_number_log(&r);
        } while (error == TOMOYO_RETRY_REQUEST);
        kfree(buf.name);
 out:
        tomoyo_read_unlock(idx);
        if (r.mode != TOMOYO_CONFIG_ENFORCING)
                error = 0;
        return error;
}

/**
 * tomoyo_check_open_permission - Check permission for "read" and "write".
 *
 * @domain: Pointer to "struct tomoyo_domain_info".
 * @path:   Pointer to "struct path".
 * @flag:   Flags for open().
 *
 * Returns 0 on success, negative value otherwise.
 */
int tomoyo_check_open_permission(struct tomoyo_domain_info *domain,
                                 const struct path *path, const int flag)
{
        const u8 acc_mode = ACC_MODE(flag);
        int error = 0;
        struct tomoyo_path_info buf;
        struct tomoyo_request_info r;
        struct tomoyo_obj_info obj = {
                .path1 = { .mnt = path->mnt, .dentry = path->dentry },
        };
        int idx;

        buf.name = NULL;
        r.mode = TOMOYO_CONFIG_DISABLED;
        idx = tomoyo_read_lock();
        if (acc_mode &&
            tomoyo_init_request_info(&r, domain, TOMOYO_MAC_FILE_OPEN)
            != TOMOYO_CONFIG_DISABLED) {
                if (!tomoyo_get_realpath(&buf, path)) {
                        error = -ENOMEM;
                        goto out;
                }
                r.obj = &obj;
                if (acc_mode & MAY_READ)
                        error = tomoyo_path_permission(&r, TOMOYO_TYPE_READ,
                                                       &buf);
                if (!error && (acc_mode & MAY_WRITE))
                        error = tomoyo_path_permission(&r, (flag & O_APPEND) ?
                                                       TOMOYO_TYPE_APPEND :
                                                       TOMOYO_TYPE_WRITE,
                                                       &buf);
        }
 out:
        kfree(buf.name);
        tomoyo_read_unlock(idx);
        if (r.mode != TOMOYO_CONFIG_ENFORCING)
                error = 0;
        return error;
}

/**
 * tomoyo_path_perm - Check permission for "unlink", "rmdir", "truncate", "symlink", "append", "chroot" and "unmount".
 *
 * @operation: Type of operation.
 * @path:      Pointer to "struct path".
 * @target:    Symlink's target if @operation is TOMOYO_TYPE_SYMLINK,
 *             NULL otherwise.
 *
 * Returns 0 on success, negative value otherwise.
 */
int tomoyo_path_perm(const u8 operation, const struct path *path, const char *target)
{
        struct tomoyo_request_info r;
        struct tomoyo_obj_info obj = {
                .path1 = { .mnt = path->mnt, .dentry = path->dentry },
        };
        int error;
        struct tomoyo_path_info buf;
        bool is_enforce;
        struct tomoyo_path_info symlink_target;
        int idx;

        if (tomoyo_init_request_info(&r, NULL, tomoyo_p2mac[operation])
            == TOMOYO_CONFIG_DISABLED)
                return 0;
        is_enforce = (r.mode == TOMOYO_CONFIG_ENFORCING);
        error = -ENOMEM;
        buf.name = NULL;
        idx = tomoyo_read_lock();
        if (!tomoyo_get_realpath(&buf, path))
                goto out;
        r.obj = &obj;
        switch (operation) {
        case TOMOYO_TYPE_RMDIR:
        case TOMOYO_TYPE_CHROOT:
                tomoyo_add_slash(&buf);
                break;
        case TOMOYO_TYPE_SYMLINK:
                symlink_target.name = tomoyo_encode(target);
                if (!symlink_target.name)
                        goto out;
                tomoyo_fill_path_info(&symlink_target);
                obj.symlink_target = &symlink_target;
                break;
        }
        error = tomoyo_path_permission(&r, operation, &buf);
        if (operation == TOMOYO_TYPE_SYMLINK)
                kfree(symlink_target.name);
 out:
        kfree(buf.name);
        tomoyo_read_unlock(idx);
        if (!is_enforce)
                error = 0;
        return error;
}

/**
 * tomoyo_mkdev_perm - Check permission for "mkblock" and "mkchar".
 *
 * @operation: Type of operation. (TOMOYO_TYPE_MKCHAR or TOMOYO_TYPE_MKBLOCK)
 * @path:      Pointer to "struct path".
 * @mode:      Create mode.
 * @dev:       Device number.
 *
 * Returns 0 on success, negative value otherwise.
 */
int tomoyo_mkdev_perm(const u8 operation, const struct path *path,
                      const unsigned int mode, unsigned int dev)
{
        struct tomoyo_request_info r;
        struct tomoyo_obj_info obj = {
                .path1 = { .mnt = path->mnt, .dentry = path->dentry },
        };
        int error = -ENOMEM;
        struct tomoyo_path_info buf;
        int idx;

        if (tomoyo_init_request_info(&r, NULL, tomoyo_pnnn2mac[operation])
            == TOMOYO_CONFIG_DISABLED)
                return 0;
        idx = tomoyo_read_lock();
        error = -ENOMEM;
        if (tomoyo_get_realpath(&buf, path)) {
                r.obj = &obj;
                dev = new_decode_dev(dev);
                r.param_type = TOMOYO_TYPE_MKDEV_ACL;
                r.param.mkdev.filename = &buf;
                r.param.mkdev.operation = operation;
                r.param.mkdev.mode = mode;
                r.param.mkdev.major = MAJOR(dev);
                r.param.mkdev.minor = MINOR(dev);
                tomoyo_check_acl(&r, tomoyo_check_mkdev_acl);
                error = tomoyo_audit_mkdev_log(&r);
                kfree(buf.name);
        }
        tomoyo_read_unlock(idx);
        if (r.mode != TOMOYO_CONFIG_ENFORCING)
                error = 0;
        return error;
}

/**
 * tomoyo_path2_perm - Check permission for "rename", "link" and "pivot_root".
 *
 * @operation: Type of operation.
 * @path1:      Pointer to "struct path".
 * @path2:      Pointer to "struct path".
 *
 * Returns 0 on success, negative value otherwise.
 */
int tomoyo_path2_perm(const u8 operation, const struct path *path1,
                      const struct path *path2)
{
        int error = -ENOMEM;
        struct tomoyo_path_info buf1;
        struct tomoyo_path_info buf2;
        struct tomoyo_request_info r;
        struct tomoyo_obj_info obj = {
                .path1 = { .mnt = path1->mnt, .dentry = path1->dentry },
                .path2 = { .mnt = path2->mnt, .dentry = path2->dentry }
        };
        int idx;

        if (tomoyo_init_request_info(&r, NULL, tomoyo_pp2mac[operation])
            == TOMOYO_CONFIG_DISABLED)
                return 0;
        buf1.name = NULL;
        buf2.name = NULL;
        idx = tomoyo_read_lock();
        if (!tomoyo_get_realpath(&buf1, path1) ||
            !tomoyo_get_realpath(&buf2, path2))
                goto out;
        switch (operation) {
        case TOMOYO_TYPE_RENAME:
        case TOMOYO_TYPE_LINK:
                if (!d_is_dir(path1->dentry))
                        break;
                fallthrough;
        case TOMOYO_TYPE_PIVOT_ROOT:
                tomoyo_add_slash(&buf1);
                tomoyo_add_slash(&buf2);
                break;
        }
        r.obj = &obj;
        r.param_type = TOMOYO_TYPE_PATH2_ACL;
        r.param.path2.operation = operation;
        r.param.path2.filename1 = &buf1;
        r.param.path2.filename2 = &buf2;
        do {
                tomoyo_check_acl(&r, tomoyo_check_path2_acl);
                error = tomoyo_audit_path2_log(&r);
        } while (error == TOMOYO_RETRY_REQUEST);
 out:
        kfree(buf1.name);
        kfree(buf2.name);
        tomoyo_read_unlock(idx);
        if (r.mode != TOMOYO_CONFIG_ENFORCING)
                error = 0;
        return error;
}

/**
 * tomoyo_same_mount_acl - Check for duplicated "struct tomoyo_mount_acl" entry.
 *
 * @a: Pointer to "struct tomoyo_acl_info".
 * @b: Pointer to "struct tomoyo_acl_info".
 *
 * Returns true if @a == @b, false otherwise.
 */
static bool tomoyo_same_mount_acl(const struct tomoyo_acl_info *a,
                                  const struct tomoyo_acl_info *b)
{
        const struct tomoyo_mount_acl *p1 = container_of(a, typeof(*p1), head);
        const struct tomoyo_mount_acl *p2 = container_of(b, typeof(*p2), head);

        return tomoyo_same_name_union(&p1->dev_name, &p2->dev_name) &&
                tomoyo_same_name_union(&p1->dir_name, &p2->dir_name) &&
                tomoyo_same_name_union(&p1->fs_type, &p2->fs_type) &&
                tomoyo_same_number_union(&p1->flags, &p2->flags);
}

/**
 * tomoyo_update_mount_acl - Write "struct tomoyo_mount_acl" list.
 *
 * @param: Pointer to "struct tomoyo_acl_param".
 *
 * Returns 0 on success, negative value otherwise.
 *
 * Caller holds tomoyo_read_lock().
 */
static int tomoyo_update_mount_acl(struct tomoyo_acl_param *param)
{
        struct tomoyo_mount_acl e = { .head.type = TOMOYO_TYPE_MOUNT_ACL };
        int error;

        if (!tomoyo_parse_name_union(param, &e.dev_name) ||
            !tomoyo_parse_name_union(param, &e.dir_name) ||
            !tomoyo_parse_name_union(param, &e.fs_type) ||
            !tomoyo_parse_number_union(param, &e.flags))
                error = -EINVAL;
        else
                error = tomoyo_update_domain(&e.head, sizeof(e), param,
                                             tomoyo_same_mount_acl, NULL);
        tomoyo_put_name_union(&e.dev_name);
        tomoyo_put_name_union(&e.dir_name);
        tomoyo_put_name_union(&e.fs_type);
        tomoyo_put_number_union(&e.flags);
        return error;
}

/**
 * tomoyo_write_file - Update file related list.
 *
 * @param: Pointer to "struct tomoyo_acl_param".
 *
 * Returns 0 on success, negative value otherwise.
 *
 * Caller holds tomoyo_read_lock().
 */
int tomoyo_write_file(struct tomoyo_acl_param *param)
{
        u16 perm = 0;
        u8 type;
        const char *operation = tomoyo_read_token(param);

        for (type = 0; type < TOMOYO_MAX_PATH_OPERATION; type++)
                if (tomoyo_permstr(operation, tomoyo_path_keyword[type]))
                        perm |= 1 << type;
        if (perm)
                return tomoyo_update_path_acl(perm, param);
        for (type = 0; type < TOMOYO_MAX_PATH2_OPERATION; type++)
                if (tomoyo_permstr(operation,
                                   tomoyo_mac_keywords[tomoyo_pp2mac[type]]))
                        perm |= 1 << type;
        if (perm)
                return tomoyo_update_path2_acl(perm, param);
        for (type = 0; type < TOMOYO_MAX_PATH_NUMBER_OPERATION; type++)
                if (tomoyo_permstr(operation,
                                   tomoyo_mac_keywords[tomoyo_pn2mac[type]]))
                        perm |= 1 << type;
        if (perm)
                return tomoyo_update_path_number_acl(perm, param);
        for (type = 0; type < TOMOYO_MAX_MKDEV_OPERATION; type++)
                if (tomoyo_permstr(operation,
                                   tomoyo_mac_keywords[tomoyo_pnnn2mac[type]]))
                        perm |= 1 << type;
        if (perm)
                return tomoyo_update_mkdev_acl(perm, param);
        if (tomoyo_permstr(operation,
                           tomoyo_mac_keywords[TOMOYO_MAC_FILE_MOUNT]))
                return tomoyo_update_mount_acl(param);
        return -EINVAL;
}



















































   30 
















   30 











   41 



   41 


































































































































































































































   22 








   22 
















   22 



























   22 








   22 





   22 
   22 






   22 



   22 


























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
// SPDX-License-Identifier: GPL-2.0
/* Linux multicast routing support
 * Common logic shared by IPv4 [ipmr] and IPv6 [ip6mr] implementation
 */

#include <linux/rhashtable.h>
#include <linux/mroute_base.h>

/* Sets everything common except 'dev', since that is done under locking */
void vif_device_init(struct vif_device *v,
                     struct net_device *dev,
                     unsigned long rate_limit,
                     unsigned char threshold,
                     unsigned short flags,
                     unsigned short get_iflink_mask)
{
        RCU_INIT_POINTER(v->dev, NULL);
        v->bytes_in = 0;
        v->bytes_out = 0;
        v->pkt_in = 0;
        v->pkt_out = 0;
        v->rate_limit = rate_limit;
        v->flags = flags;
        v->threshold = threshold;
        if (v->flags & get_iflink_mask)
                v->link = dev_get_iflink(dev);
        else
                v->link = dev->ifindex;
}

static void __mr_free_table(struct work_struct *work)
{
        struct mr_table *mrt = container_of(to_rcu_work(work),
                                            struct mr_table, work);

        rhltable_destroy(&mrt->mfc_hash);
        kfree(mrt);
}

void mr_table_free(struct mr_table *mrt)
{
        queue_rcu_work(system_unbound_wq, &mrt->work);
}

struct mr_table *
mr_table_alloc(struct net *net, u32 id,
               struct mr_table_ops *ops,
               void (*expire_func)(struct timer_list *t),
               void (*table_set)(struct mr_table *mrt,
                                 struct net *net))
{
        struct mr_table *mrt;
        int err;

        mrt = kzalloc_obj(*mrt);
        if (!mrt)
                return ERR_PTR(-ENOMEM);
        mrt->id = id;
        write_pnet(&mrt->net, net);

        mrt->ops = *ops;
        err = rhltable_init(&mrt->mfc_hash, mrt->ops.rht_params);
        if (err) {
                kfree(mrt);
                return ERR_PTR(err);
        }

        INIT_RCU_WORK(&mrt->work, __mr_free_table);
        INIT_LIST_HEAD(&mrt->mfc_cache_list);
        INIT_LIST_HEAD(&mrt->mfc_unres_queue);

        timer_setup(&mrt->ipmr_expire_timer, expire_func, 0);

        mrt->mroute_reg_vif_num = -1;
        table_set(mrt, net);
        return mrt;
}

void *mr_mfc_find_parent(struct mr_table *mrt, void *hasharg, int parent)
{
        struct rhlist_head *tmp, *list;
        struct mr_mfc *c;

        list = rhltable_lookup(&mrt->mfc_hash, hasharg, *mrt->ops.rht_params);
        rhl_for_each_entry_rcu(c, tmp, list, mnode)
                if (parent == -1 || parent == c->mfc_parent)
                        return c;

        return NULL;
}

void *mr_mfc_find_any_parent(struct mr_table *mrt, int vifi)
{
        struct rhlist_head *tmp, *list;
        struct mr_mfc *c;

        list = rhltable_lookup(&mrt->mfc_hash, mrt->ops.cmparg_any,
                               *mrt->ops.rht_params);
        rhl_for_each_entry_rcu(c, tmp, list, mnode)
                if (c->mfc_un.res.ttls[vifi] < 255)
                        return c;

        return NULL;
}

void *mr_mfc_find_any(struct mr_table *mrt, int vifi, void *hasharg)
{
        struct rhlist_head *tmp, *list;
        struct mr_mfc *c, *proxy;

        list = rhltable_lookup(&mrt->mfc_hash, hasharg, *mrt->ops.rht_params);
        rhl_for_each_entry_rcu(c, tmp, list, mnode) {
                if (c->mfc_un.res.ttls[vifi] < 255)
                        return c;

                /* It's ok if the vifi is part of the static tree */
                proxy = mr_mfc_find_any_parent(mrt, c->mfc_parent);
                if (proxy && proxy->mfc_un.res.ttls[vifi] < 255)
                        return c;
        }

        return mr_mfc_find_any_parent(mrt, vifi);
}

#ifdef CONFIG_PROC_FS
void *mr_vif_seq_idx(struct net *net, struct mr_vif_iter *iter, loff_t pos)
{
        struct mr_table *mrt = iter->mrt;

        for (iter->ct = 0; iter->ct < mrt->maxvif; ++iter->ct) {
                if (!VIF_EXISTS(mrt, iter->ct))
                        continue;
                if (pos-- == 0)
                        return &mrt->vif_table[iter->ct];
        }
        return NULL;
}

void *mr_vif_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct mr_vif_iter *iter = seq->private;
        struct net *net = seq_file_net(seq);
        struct mr_table *mrt = iter->mrt;

        ++*pos;
        if (v == SEQ_START_TOKEN)
                return mr_vif_seq_idx(net, iter, 0);

        while (++iter->ct < mrt->maxvif) {
                if (!VIF_EXISTS(mrt, iter->ct))
                        continue;
                return &mrt->vif_table[iter->ct];
        }
        return NULL;
}

void *mr_mfc_seq_idx(struct net *net,
                     struct mr_mfc_iter *it, loff_t pos)
{
        struct mr_table *mrt = it->mrt;
        struct mr_mfc *mfc;

        rcu_read_lock();
        it->cache = &mrt->mfc_cache_list;
        list_for_each_entry_rcu(mfc, &mrt->mfc_cache_list, list)
                if (pos-- == 0)
                        return mfc;
        rcu_read_unlock();

        spin_lock_bh(it->lock);
        it->cache = &mrt->mfc_unres_queue;
        list_for_each_entry(mfc, it->cache, list)
                if (pos-- == 0)
                        return mfc;
        spin_unlock_bh(it->lock);

        it->cache = NULL;
        return NULL;
}

void *mr_mfc_seq_next(struct seq_file *seq, void *v,
                      loff_t *pos)
{
        struct mr_mfc_iter *it = seq->private;
        struct net *net = seq_file_net(seq);
        struct mr_table *mrt = it->mrt;
        struct mr_mfc *c = v;

        ++*pos;

        if (v == SEQ_START_TOKEN)
                return mr_mfc_seq_idx(net, seq->private, 0);

        if (c->list.next != it->cache)
                return list_entry(c->list.next, struct mr_mfc, list);

        if (it->cache == &mrt->mfc_unres_queue)
                goto end_of_list;

        /* exhausted cache_array, show unresolved */
        rcu_read_unlock();
        it->cache = &mrt->mfc_unres_queue;

        spin_lock_bh(it->lock);
        if (!list_empty(it->cache))
                return list_first_entry(it->cache, struct mr_mfc, list);

end_of_list:
        spin_unlock_bh(it->lock);
        it->cache = NULL;

        return NULL;
}
#endif

int mr_fill_mroute(struct mr_table *mrt, struct sk_buff *skb,
                   struct mr_mfc *c, struct rtmsg *rtm)
{
        struct net_device *vif_dev;
        struct rta_mfc_stats mfcs;
        struct nlattr *mp_attr;
        struct rtnexthop *nhp;
        unsigned long lastuse;
        int ct;

        /* If cache is unresolved, don't try to parse IIF and OIF */
        if (c->mfc_parent >= MAXVIFS) {
                rtm->rtm_flags |= RTNH_F_UNRESOLVED;
                return -ENOENT;
        }

        rcu_read_lock();
        vif_dev = rcu_dereference(mrt->vif_table[c->mfc_parent].dev);
        if (vif_dev && nla_put_u32(skb, RTA_IIF, READ_ONCE(vif_dev->ifindex)) < 0) {
                rcu_read_unlock();
                return -EMSGSIZE;
        }
        rcu_read_unlock();

        if (c->mfc_flags & MFC_OFFLOAD)
                rtm->rtm_flags |= RTNH_F_OFFLOAD;

        mp_attr = nla_nest_start_noflag(skb, RTA_MULTIPATH);
        if (!mp_attr)
                return -EMSGSIZE;

        rcu_read_lock();
        for (ct = c->mfc_un.res.minvif; ct < c->mfc_un.res.maxvif; ct++) {
                struct vif_device *vif = &mrt->vif_table[ct];

                vif_dev = rcu_dereference(vif->dev);
                if (vif_dev && c->mfc_un.res.ttls[ct] < 255) {

                        nhp = nla_reserve_nohdr(skb, sizeof(*nhp));
                        if (!nhp) {
                                rcu_read_unlock();
                                nla_nest_cancel(skb, mp_attr);
                                return -EMSGSIZE;
                        }

                        nhp->rtnh_flags = 0;
                        nhp->rtnh_hops = c->mfc_un.res.ttls[ct];
                        nhp->rtnh_ifindex = READ_ONCE(vif_dev->ifindex);
                        nhp->rtnh_len = sizeof(*nhp);
                }
        }
        rcu_read_unlock();

        nla_nest_end(skb, mp_attr);

        lastuse = READ_ONCE(c->mfc_un.res.lastuse);
        lastuse = time_after_eq(jiffies, lastuse) ? jiffies - lastuse : 0;

        mfcs.mfcs_packets = atomic_long_read(&c->mfc_un.res.pkt);
        mfcs.mfcs_bytes = atomic_long_read(&c->mfc_un.res.bytes);
        mfcs.mfcs_wrong_if = atomic_long_read(&c->mfc_un.res.wrong_if);
        if (nla_put_64bit(skb, RTA_MFC_STATS, sizeof(mfcs), &mfcs, RTA_PAD) ||
            nla_put_u64_64bit(skb, RTA_EXPIRES, jiffies_to_clock_t(lastuse),
                              RTA_PAD))
                return -EMSGSIZE;

        rtm->rtm_type = RTN_MULTICAST;
        return 1;
}

static bool mr_mfc_uses_dev(const struct mr_table *mrt,
                            const struct mr_mfc *c,
                            const struct net_device *dev)
{
        int ct;

        for (ct = c->mfc_un.res.minvif; ct < c->mfc_un.res.maxvif; ct++) {
                const struct net_device *vif_dev;
                const struct vif_device *vif;

                vif = &mrt->vif_table[ct];
                vif_dev = rcu_access_pointer(vif->dev);
                if (vif_dev && c->mfc_un.res.ttls[ct] < 255 &&
                    vif_dev == dev)
                        return true;
        }
        return false;
}

int mr_table_dump(struct mr_table *mrt, struct sk_buff *skb,
                  struct netlink_callback *cb,
                  int (*fill)(struct mr_table *mrt, struct sk_buff *skb,
                              u32 portid, u32 seq, struct mr_mfc *c,
                              int cmd, int flags),
                  spinlock_t *lock, struct fib_dump_filter *filter)
{
        unsigned int e = 0, s_e = cb->args[1];
        unsigned int flags = NLM_F_MULTI;
        struct mr_mfc *mfc;
        int err;

        if (filter->filter_set)
                flags |= NLM_F_DUMP_FILTERED;

        list_for_each_entry_rcu(mfc, &mrt->mfc_cache_list, list,
                                lockdep_rtnl_is_held()) {
                if (e < s_e)
                        goto next_entry;
                if (filter->dev &&
                    !mr_mfc_uses_dev(mrt, mfc, filter->dev))
                        goto next_entry;

                err = fill(mrt, skb, NETLINK_CB(cb->skb).portid,
                           cb->nlh->nlmsg_seq, mfc, RTM_NEWROUTE, flags);
                if (err < 0)
                        goto out;
next_entry:
                e++;
        }

        spin_lock_bh(lock);
        list_for_each_entry(mfc, &mrt->mfc_unres_queue, list) {
                if (e < s_e)
                        goto next_entry2;

                err = fill(mrt, skb, NETLINK_CB(cb->skb).portid,
                           cb->nlh->nlmsg_seq, mfc, RTM_NEWROUTE, flags);
                if (err < 0) {
                        spin_unlock_bh(lock);
                        goto out;
                }
next_entry2:
                e++;
        }
        spin_unlock_bh(lock);
        err = 0;
out:
        cb->args[1] = e;
        return err;
}

int mr_rtm_dumproute(struct sk_buff *skb, struct netlink_callback *cb,
                     struct mr_table *(*iter)(struct net *net,
                                              struct mr_table *mrt),
                     int (*fill)(struct mr_table *mrt,
                                 struct sk_buff *skb,
                                 u32 portid, u32 seq, struct mr_mfc *c,
                                 int cmd, int flags),
                     spinlock_t *lock, struct fib_dump_filter *filter)
{
        unsigned int t = 0, s_t = cb->args[0];
        struct net *net = sock_net(skb->sk);
        struct mr_table *mrt;
        int err;

        /* multicast does not track protocol or have route type other
         * than RTN_MULTICAST
         */
        if (filter->filter_set) {
                if (filter->protocol || filter->flags ||
                    (filter->rt_type && filter->rt_type != RTN_MULTICAST))
                        return skb->len;
        }

        rcu_read_lock();
        for (mrt = iter(net, NULL); mrt; mrt = iter(net, mrt)) {
                if (t < s_t)
                        goto next_table;

                err = mr_table_dump(mrt, skb, cb, fill, lock, filter);
                if (err < 0)
                        break;
                cb->args[1] = 0;
next_table:
                t++;
        }
        rcu_read_unlock();

        cb->args[0] = t;

        return skb->len;
}

int mr_dump(struct net *net, struct notifier_block *nb, unsigned short family,
            int (*rules_dump)(struct net *net,
                              struct notifier_block *nb,
                              struct netlink_ext_ack *extack),
            struct mr_table *(*mr_iter)(struct net *net,
                                        struct mr_table *mrt),
            struct netlink_ext_ack *extack)
{
        struct mr_table *mrt;
        int err;

        err = rules_dump(net, nb, extack);
        if (err)
                return err;

        for (mrt = mr_iter(net, NULL); mrt; mrt = mr_iter(net, mrt)) {
                struct vif_device *v = &mrt->vif_table[0];
                struct net_device *vif_dev;
                struct mr_mfc *mfc;
                int vifi;

                /* Notifiy on table VIF entries */
                rcu_read_lock();
                for (vifi = 0; vifi < mrt->maxvif; vifi++, v++) {
                        vif_dev = rcu_dereference(v->dev);
                        if (!vif_dev)
                                continue;

                        err = mr_call_vif_notifier(nb, family,
                                                   FIB_EVENT_VIF_ADD, v,
                                                   vif_dev, vifi,
                                                   mrt->id, extack);
                        if (err)
                                break;
                }
                rcu_read_unlock();

                if (err)
                        return err;

                /* Notify on table MFC entries */
                list_for_each_entry_rcu(mfc, &mrt->mfc_cache_list, list) {
                        err = mr_call_mfc_notifier(nb, family,
                                                   FIB_EVENT_ENTRY_ADD,
                                                   mfc, mrt->id, extack);
                        if (err)
                                return err;
                }
        }

        return 0;
}





















































































































































    3 








































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2020 Laura Garcia Liebana <nevola@gmail.com>
 * Copyright (c) 2020 Jose M. Guisado <guigom@riseup.net>
 */

#include <linux/etherdevice.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nft_reject.h>
#include <net/netfilter/ipv4/nf_reject.h>
#include <net/netfilter/ipv6/nf_reject.h>

static void nft_reject_queue_xmit(struct sk_buff *nskb, struct sk_buff *oldskb)
{
        dev_hard_header(nskb, nskb->dev, ntohs(oldskb->protocol),
                        eth_hdr(oldskb)->h_source, eth_hdr(oldskb)->h_dest,
                        nskb->len);
        dev_queue_xmit(nskb);
}

static void nft_reject_netdev_send_v4_tcp_reset(struct net *net,
                                                struct sk_buff *oldskb,
                                                const struct net_device *dev,
                                                int hook)
{
        struct sk_buff *nskb;

        nskb = nf_reject_skb_v4_tcp_reset(net, oldskb, dev, hook);
        if (!nskb)
                return;

        nft_reject_queue_xmit(nskb, oldskb);
}

static void nft_reject_netdev_send_v4_unreach(struct net *net,
                                              struct sk_buff *oldskb,
                                              const struct net_device *dev,
                                              int hook, u8 code)
{
        struct sk_buff *nskb;

        nskb = nf_reject_skb_v4_unreach(net, oldskb, dev, hook, code);
        if (!nskb)
                return;

        nft_reject_queue_xmit(nskb, oldskb);
}

static void nft_reject_netdev_send_v6_tcp_reset(struct net *net,
                                                struct sk_buff *oldskb,
                                                const struct net_device *dev,
                                                int hook)
{
        struct sk_buff *nskb;

        nskb = nf_reject_skb_v6_tcp_reset(net, oldskb, dev, hook);
        if (!nskb)
                return;

        nft_reject_queue_xmit(nskb, oldskb);
}


static void nft_reject_netdev_send_v6_unreach(struct net *net,
                                              struct sk_buff *oldskb,
                                              const struct net_device *dev,
                                              int hook, u8 code)
{
        struct sk_buff *nskb;

        nskb = nf_reject_skb_v6_unreach(net, oldskb, dev, hook, code);
        if (!nskb)
                return;

        nft_reject_queue_xmit(nskb, oldskb);
}

static void nft_reject_netdev_eval(const struct nft_expr *expr,
                                   struct nft_regs *regs,
                                   const struct nft_pktinfo *pkt)
{
        struct ethhdr *eth = eth_hdr(pkt->skb);
        struct nft_reject *priv = nft_expr_priv(expr);
        const unsigned char *dest = eth->h_dest;

        if (is_broadcast_ether_addr(dest) ||
            is_multicast_ether_addr(dest))
                goto out;

        switch (eth->h_proto) {
        case htons(ETH_P_IP):
                switch (priv->type) {
                case NFT_REJECT_ICMP_UNREACH:
                        nft_reject_netdev_send_v4_unreach(nft_net(pkt), pkt->skb,
                                                          nft_in(pkt),
                                                          nft_hook(pkt),
                                                          priv->icmp_code);
                        break;
                case NFT_REJECT_TCP_RST:
                        nft_reject_netdev_send_v4_tcp_reset(nft_net(pkt), pkt->skb,
                                                            nft_in(pkt),
                                                            nft_hook(pkt));
                        break;
                case NFT_REJECT_ICMPX_UNREACH:
                        nft_reject_netdev_send_v4_unreach(nft_net(pkt), pkt->skb,
                                                          nft_in(pkt),
                                                          nft_hook(pkt),
                                                          nft_reject_icmp_code(priv->icmp_code));
                        break;
                }
                break;
        case htons(ETH_P_IPV6):
                switch (priv->type) {
                case NFT_REJECT_ICMP_UNREACH:
                        nft_reject_netdev_send_v6_unreach(nft_net(pkt), pkt->skb,
                                                          nft_in(pkt),
                                                          nft_hook(pkt),
                                                          priv->icmp_code);
                        break;
                case NFT_REJECT_TCP_RST:
                        nft_reject_netdev_send_v6_tcp_reset(nft_net(pkt), pkt->skb,
                                                            nft_in(pkt),
                                                            nft_hook(pkt));
                        break;
                case NFT_REJECT_ICMPX_UNREACH:
                        nft_reject_netdev_send_v6_unreach(nft_net(pkt), pkt->skb,
                                                          nft_in(pkt),
                                                          nft_hook(pkt),
                                                          nft_reject_icmpv6_code(priv->icmp_code));
                        break;
                }
                break;
        default:
                /* No explicit way to reject this protocol, drop it. */
                break;
        }
out:
        regs->verdict.code = NF_DROP;
}

static int nft_reject_netdev_validate(const struct nft_ctx *ctx,
                                      const struct nft_expr *expr)
{
        return nft_chain_validate_hooks(ctx->chain, (1 << NF_NETDEV_INGRESS));
}

static struct nft_expr_type nft_reject_netdev_type;
static const struct nft_expr_ops nft_reject_netdev_ops = {
        .type                = &nft_reject_netdev_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_reject)),
        .eval                = nft_reject_netdev_eval,
        .init                = nft_reject_init,
        .dump                = nft_reject_dump,
        .validate        = nft_reject_netdev_validate,
};

static struct nft_expr_type nft_reject_netdev_type __read_mostly = {
        .family                = NFPROTO_NETDEV,
        .name                = "reject",
        .ops                = &nft_reject_netdev_ops,
        .policy                = nft_reject_policy,
        .maxattr        = NFTA_REJECT_MAX,
        .owner                = THIS_MODULE,
};

static int __init nft_reject_netdev_module_init(void)
{
        return nft_register_expr(&nft_reject_netdev_type);
}

static void __exit nft_reject_netdev_module_exit(void)
{
        nft_unregister_expr(&nft_reject_netdev_type);
}

module_init(nft_reject_netdev_module_init);
module_exit(nft_reject_netdev_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Laura Garcia Liebana <nevola@gmail.com>");
MODULE_AUTHOR("Jose M. Guisado <guigom@riseup.net>");
MODULE_DESCRIPTION("Reject packets from netdev via nftables");
MODULE_ALIAS_NFT_AF_EXPR(5, "reject");































































































































































































   32 

















































































































   32 








   32 





   32 














   32 



   32 
   32 




   32 









   32 


































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
// SPDX-License-Identifier: GPL-2.0-or-later
/* mpihelp-div.c  -  MPI helper functions
 *        Copyright (C) 1994, 1996 Free Software Foundation, Inc.
 *        Copyright (C) 1998, 1999 Free Software Foundation, Inc.
 *
 * This file is part of GnuPG.
 *
 * Note: This code is heavily based on the GNU MP Library.
 *         Actually it's the same code with only minor changes in the
 *         way the data is stored; this is to support the abstraction
 *         of an optional secure memory allocation which may be used
 *         to avoid revealing of sensitive data due to paging etc.
 *         The GNU MP Library itself is published under the LGPL;
 *         however I decided to publish this code under the plain GPL.
 */

#include "mpi-internal.h"
#include "longlong.h"

#ifndef UMUL_TIME
#define UMUL_TIME 1
#endif
#ifndef UDIV_TIME
#define UDIV_TIME UMUL_TIME
#endif


mpi_limb_t
mpihelp_mod_1(mpi_ptr_t dividend_ptr, mpi_size_t dividend_size,
                        mpi_limb_t divisor_limb)
{
        mpi_size_t i;
        mpi_limb_t n1, n0, r;
        mpi_limb_t dummy __maybe_unused;

        /* Botch: Should this be handled at all?  Rely on callers?        */
        if (!dividend_size)
                return 0;

        /* If multiplication is much faster than division, and the
         * dividend is large, pre-invert the divisor, and use
         * only multiplications in the inner loop.
         *
         * This test should be read:
         *         Does it ever help to use udiv_qrnnd_preinv?
         *           && Does what we save compensate for the inversion overhead?
         */
        if (UDIV_TIME > (2 * UMUL_TIME + 6)
                        && (UDIV_TIME - (2 * UMUL_TIME + 6)) * dividend_size > UDIV_TIME) {
                int normalization_steps;

                normalization_steps = count_leading_zeros(divisor_limb);
                if (normalization_steps) {
                        mpi_limb_t divisor_limb_inverted;

                        divisor_limb <<= normalization_steps;

                        /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB.  The
                         * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the
                         * most significant bit (with weight 2**N) implicit.
                         *
                         * Special case for DIVISOR_LIMB == 100...000.
                         */
                        if (!(divisor_limb << 1))
                                divisor_limb_inverted = ~(mpi_limb_t)0;
                        else
                                udiv_qrnnd(divisor_limb_inverted, dummy,
                                                -divisor_limb, 0, divisor_limb);

                        n1 = dividend_ptr[dividend_size - 1];
                        r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps);

                        /* Possible optimization:
                         * if (r == 0
                         * && divisor_limb > ((n1 << normalization_steps)
                         *                       | (dividend_ptr[dividend_size - 2] >> ...)))
                         * ...one division less...
                         */
                        for (i = dividend_size - 2; i >= 0; i--) {
                                n0 = dividend_ptr[i];
                                UDIV_QRNND_PREINV(dummy, r, r,
                                                ((n1 << normalization_steps)
                                                 | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))),
                                                divisor_limb, divisor_limb_inverted);
                                n1 = n0;
                        }
                        UDIV_QRNND_PREINV(dummy, r, r,
                                        n1 << normalization_steps,
                                        divisor_limb, divisor_limb_inverted);
                        return r >> normalization_steps;
                } else {
                        mpi_limb_t divisor_limb_inverted;

                        /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB.  The
                         * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the
                         * most significant bit (with weight 2**N) implicit.
                         *
                         * Special case for DIVISOR_LIMB == 100...000.
                         */
                        if (!(divisor_limb << 1))
                                divisor_limb_inverted = ~(mpi_limb_t)0;
                        else
                                udiv_qrnnd(divisor_limb_inverted, dummy,
                                                -divisor_limb, 0, divisor_limb);

                        i = dividend_size - 1;
                        r = dividend_ptr[i];

                        if (r >= divisor_limb)
                                r = 0;
                        else
                                i--;

                        for ( ; i >= 0; i--) {
                                n0 = dividend_ptr[i];
                                UDIV_QRNND_PREINV(dummy, r, r,
                                                n0, divisor_limb, divisor_limb_inverted);
                        }
                        return r;
                }
        } else {
                if (UDIV_NEEDS_NORMALIZATION) {
                        int normalization_steps;

                        normalization_steps = count_leading_zeros(divisor_limb);
                        if (normalization_steps) {
                                divisor_limb <<= normalization_steps;

                                n1 = dividend_ptr[dividend_size - 1];
                                r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps);

                                /* Possible optimization:
                                 * if (r == 0
                                 * && divisor_limb > ((n1 << normalization_steps)
                                 *                   | (dividend_ptr[dividend_size - 2] >> ...)))
                                 * ...one division less...
                                 */
                                for (i = dividend_size - 2; i >= 0; i--) {
                                        n0 = dividend_ptr[i];
                                        udiv_qrnnd(dummy, r, r,
                                                ((n1 << normalization_steps)
                                                 | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))),
                                                divisor_limb);
                                        n1 = n0;
                                }
                                udiv_qrnnd(dummy, r, r,
                                                n1 << normalization_steps,
                                                divisor_limb);
                                return r >> normalization_steps;
                        }
                }
                /* No normalization needed, either because udiv_qrnnd doesn't require
                 * it, or because DIVISOR_LIMB is already normalized.
                 */
                i = dividend_size - 1;
                r = dividend_ptr[i];

                if (r >= divisor_limb)
                        r = 0;
                else
                        i--;

                for (; i >= 0; i--) {
                        n0 = dividend_ptr[i];
                        udiv_qrnnd(dummy, r, r, n0, divisor_limb);
                }
                return r;
        }
}

/* Divide num (NP/NSIZE) by den (DP/DSIZE) and write
 * the NSIZE-DSIZE least significant quotient limbs at QP
 * and the DSIZE long remainder at NP.        If QEXTRA_LIMBS is
 * non-zero, generate that many fraction bits and append them after the
 * other quotient limbs.
 * Return the most significant limb of the quotient, this is always 0 or 1.
 *
 * Preconditions:
 * 0. NSIZE >= DSIZE.
 * 1. The most significant bit of the divisor must be set.
 * 2. QP must either not overlap with the input operands at all, or
 *    QP + DSIZE >= NP must hold true.        (This means that it's
 *    possible to put the quotient in the high part of NUM, right after the
 *    remainder in NUM.
 * 3. NSIZE >= DSIZE, even if QEXTRA_LIMBS is non-zero.
 */

mpi_limb_t
mpihelp_divrem(mpi_ptr_t qp, mpi_size_t qextra_limbs,
               mpi_ptr_t np, mpi_size_t nsize, mpi_ptr_t dp, mpi_size_t dsize)
{
        mpi_limb_t most_significant_q_limb = 0;

        switch (dsize) {
        case 0:
                /* We are asked to divide by zero, so go ahead and do it!  (To make
                   the compiler not remove this statement, return the value.)  */
                /*
                 * existing clients of this function have been modified
                 * not to call it with dsize == 0, so this should not happen
                 */
                return 1 / dsize;

        case 1:
                {
                        mpi_size_t i;
                        mpi_limb_t n1;
                        mpi_limb_t d;

                        d = dp[0];
                        n1 = np[nsize - 1];

                        if (n1 >= d) {
                                n1 -= d;
                                most_significant_q_limb = 1;
                        }

                        qp += qextra_limbs;
                        for (i = nsize - 2; i >= 0; i--)
                                udiv_qrnnd(qp[i], n1, n1, np[i], d);
                        qp -= qextra_limbs;

                        for (i = qextra_limbs - 1; i >= 0; i--)
                                udiv_qrnnd(qp[i], n1, n1, 0, d);

                        np[0] = n1;
                }
                break;

        case 2:
                {
                        mpi_size_t i;
                        mpi_limb_t n1, n0, n2;
                        mpi_limb_t d1, d0;

                        np += nsize - 2;
                        d1 = dp[1];
                        d0 = dp[0];
                        n1 = np[1];
                        n0 = np[0];

                        if (n1 >= d1 && (n1 > d1 || n0 >= d0)) {
                                sub_ddmmss(n1, n0, n1, n0, d1, d0);
                                most_significant_q_limb = 1;
                        }

                        for (i = qextra_limbs + nsize - 2 - 1; i >= 0; i--) {
                                mpi_limb_t q;
                                mpi_limb_t r;

                                if (i >= qextra_limbs)
                                        np--;
                                else
                                        np[0] = 0;

                                if (n1 == d1) {
                                        /* Q should be either 111..111 or 111..110.  Need special
                                         * treatment of this rare case as normal division would
                                         * give overflow.  */
                                        q = ~(mpi_limb_t) 0;

                                        r = n0 + d1;
                                        if (r < d1) {        /* Carry in the addition? */
                                                add_ssaaaa(n1, n0, r - d0,
                                                           np[0], 0, d0);
                                                qp[i] = q;
                                                continue;
                                        }
                                        n1 = d0 - (d0 != 0 ? 1 : 0);
                                        n0 = -d0;
                                } else {
                                        udiv_qrnnd(q, r, n1, n0, d1);
                                        umul_ppmm(n1, n0, d0, q);
                                }

                                n2 = np[0];
q_test:
                                if (n1 > r || (n1 == r && n0 > n2)) {
                                        /* The estimated Q was too large.  */
                                        q--;
                                        sub_ddmmss(n1, n0, n1, n0, 0, d0);
                                        r += d1;
                                        if (r >= d1)        /* If not carry, test Q again.  */
                                                goto q_test;
                                }

                                qp[i] = q;
                                sub_ddmmss(n1, n0, r, n2, n1, n0);
                        }
                        np[1] = n1;
                        np[0] = n0;
                }
                break;

        default:
                {
                        mpi_size_t i;
                        mpi_limb_t dX, d1, n0;

                        np += nsize - dsize;
                        dX = dp[dsize - 1];
                        d1 = dp[dsize - 2];
                        n0 = np[dsize - 1];

                        if (n0 >= dX) {
                                if (n0 > dX
                                    || mpihelp_cmp(np, dp, dsize - 1) >= 0) {
                                        mpihelp_sub_n(np, np, dp, dsize);
                                        n0 = np[dsize - 1];
                                        most_significant_q_limb = 1;
                                }
                        }

                        for (i = qextra_limbs + nsize - dsize - 1; i >= 0; i--) {
                                mpi_limb_t q;
                                mpi_limb_t n1, n2;
                                mpi_limb_t cy_limb;

                                if (i >= qextra_limbs) {
                                        np--;
                                        n2 = np[dsize];
                                } else {
                                        n2 = np[dsize - 1];
                                        MPN_COPY_DECR(np + 1, np, dsize - 1);
                                        np[0] = 0;
                                }

                                if (n0 == dX) {
                                        /* This might over-estimate q, but it's probably not worth
                                         * the extra code here to find out.  */
                                        q = ~(mpi_limb_t) 0;
                                } else {
                                        mpi_limb_t r;

                                        udiv_qrnnd(q, r, n0, np[dsize - 1], dX);
                                        umul_ppmm(n1, n0, d1, q);

                                        while (n1 > r
                                               || (n1 == r
                                                   && n0 > np[dsize - 2])) {
                                                q--;
                                                r += dX;
                                                if (r < dX)        /* I.e. "carry in previous addition?" */
                                                        break;
                                                n1 -= n0 < d1;
                                                n0 -= d1;
                                        }
                                }

                                /* Possible optimization: We already have (q * n0) and (1 * n1)
                                 * after the calculation of q.  Taking advantage of that, we
                                 * could make this loop make two iterations less.  */
                                cy_limb = mpihelp_submul_1(np, dp, dsize, q);

                                if (n2 != cy_limb) {
                                        mpihelp_add_n(np, np, dp, dsize);
                                        q--;
                                }

                                qp[i] = q;
                                n0 = np[dsize - 1];
                        }
                }
        }

        return most_significant_q_limb;
}

/****************
 * Divide (DIVIDEND_PTR,,DIVIDEND_SIZE) by DIVISOR_LIMB.
 * Write DIVIDEND_SIZE limbs of quotient at QUOT_PTR.
 * Return the single-limb remainder.
 * There are no constraints on the value of the divisor.
 *
 * QUOT_PTR and DIVIDEND_PTR might point to the same limb.
 */

mpi_limb_t
mpihelp_divmod_1(mpi_ptr_t quot_ptr,
                mpi_ptr_t dividend_ptr, mpi_size_t dividend_size,
                mpi_limb_t divisor_limb)
{
        mpi_size_t i;
        mpi_limb_t n1, n0, r;
        mpi_limb_t dummy __maybe_unused;

        if (!dividend_size)
                return 0;

        /* If multiplication is much faster than division, and the
         * dividend is large, pre-invert the divisor, and use
         * only multiplications in the inner loop.
         *
         * This test should be read:
         * Does it ever help to use udiv_qrnnd_preinv?
         * && Does what we save compensate for the inversion overhead?
         */
        if (UDIV_TIME > (2 * UMUL_TIME + 6)
                        && (UDIV_TIME - (2 * UMUL_TIME + 6)) * dividend_size > UDIV_TIME) {
                int normalization_steps;

                normalization_steps = count_leading_zeros(divisor_limb);
                if (normalization_steps) {
                        mpi_limb_t divisor_limb_inverted;

                        divisor_limb <<= normalization_steps;

                        /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB.  The
                         * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the
                         * most significant bit (with weight 2**N) implicit.
                         */
                        /* Special case for DIVISOR_LIMB == 100...000.  */
                        if (!(divisor_limb << 1))
                                divisor_limb_inverted = ~(mpi_limb_t)0;
                        else
                                udiv_qrnnd(divisor_limb_inverted, dummy,
                                                -divisor_limb, 0, divisor_limb);

                        n1 = dividend_ptr[dividend_size - 1];
                        r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps);

                        /* Possible optimization:
                         * if (r == 0
                         * && divisor_limb > ((n1 << normalization_steps)
                         *                       | (dividend_ptr[dividend_size - 2] >> ...)))
                         * ...one division less...
                         */
                        for (i = dividend_size - 2; i >= 0; i--) {
                                n0 = dividend_ptr[i];
                                UDIV_QRNND_PREINV(quot_ptr[i + 1], r, r,
                                                ((n1 << normalization_steps)
                                                 | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))),
                                                divisor_limb, divisor_limb_inverted);
                                n1 = n0;
                        }
                        UDIV_QRNND_PREINV(quot_ptr[0], r, r,
                                        n1 << normalization_steps,
                                        divisor_limb, divisor_limb_inverted);
                        return r >> normalization_steps;
                } else {
                        mpi_limb_t divisor_limb_inverted;

                        /* Compute (2**2N - 2**N * DIVISOR_LIMB) / DIVISOR_LIMB.  The
                         * result is a (N+1)-bit approximation to 1/DIVISOR_LIMB, with the
                         * most significant bit (with weight 2**N) implicit.
                         */
                        /* Special case for DIVISOR_LIMB == 100...000.  */
                        if (!(divisor_limb << 1))
                                divisor_limb_inverted = ~(mpi_limb_t) 0;
                        else
                                udiv_qrnnd(divisor_limb_inverted, dummy,
                                                -divisor_limb, 0, divisor_limb);

                        i = dividend_size - 1;
                        r = dividend_ptr[i];

                        if (r >= divisor_limb)
                                r = 0;
                        else
                                quot_ptr[i--] = 0;

                        for ( ; i >= 0; i--) {
                                n0 = dividend_ptr[i];
                                UDIV_QRNND_PREINV(quot_ptr[i], r, r,
                                                n0, divisor_limb, divisor_limb_inverted);
                        }
                        return r;
                }
        } else {
                if (UDIV_NEEDS_NORMALIZATION) {
                        int normalization_steps;

                        normalization_steps = count_leading_zeros(divisor_limb);
                        if (normalization_steps) {
                                divisor_limb <<= normalization_steps;

                                n1 = dividend_ptr[dividend_size - 1];
                                r = n1 >> (BITS_PER_MPI_LIMB - normalization_steps);

                                /* Possible optimization:
                                 * if (r == 0
                                 * && divisor_limb > ((n1 << normalization_steps)
                                 *                   | (dividend_ptr[dividend_size - 2] >> ...)))
                                 * ...one division less...
                                 */
                                for (i = dividend_size - 2; i >= 0; i--) {
                                        n0 = dividend_ptr[i];
                                        udiv_qrnnd(quot_ptr[i + 1], r, r,
                                                ((n1 << normalization_steps)
                                                 | (n0 >> (BITS_PER_MPI_LIMB - normalization_steps))),
                                                divisor_limb);
                                        n1 = n0;
                                }
                                udiv_qrnnd(quot_ptr[0], r, r,
                                                n1 << normalization_steps,
                                                divisor_limb);
                                return r >> normalization_steps;
                        }
                }
                /* No normalization needed, either because udiv_qrnnd doesn't require
                 * it, or because DIVISOR_LIMB is already normalized.
                 */
                i = dividend_size - 1;
                r = dividend_ptr[i];

                if (r >= divisor_limb)
                        r = 0;
                else
                        quot_ptr[i--] = 0;

                for (; i >= 0; i--) {
                        n0 = dividend_ptr[i];
                        udiv_qrnnd(quot_ptr[i], r, r, n0, divisor_limb);
                }
                return r;
        }
}







































   51 



   51 

   51 

   51 

   51 














































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
// SPDX-License-Identifier: GPL-2.0-only
/*
 * File: sysctl.c
 *
 * Phonet /proc/sys/net/phonet interface implementation
 *
 * Copyright (C) 2008 Nokia Corporation.
 *
 * Author: Rémi Denis-Courmont
 */

#include <linux/seqlock.h>
#include <linux/sysctl.h>
#include <linux/errno.h>
#include <linux/init.h>

#include <net/sock.h>
#include <linux/phonet.h>
#include <net/phonet/phonet.h>

#define DYNAMIC_PORT_MIN        0x40
#define DYNAMIC_PORT_MAX        0x7f

static DEFINE_SEQLOCK(local_port_range_lock);
static int local_port_range_min[2] = {0, 0};
static int local_port_range_max[2] = {1023, 1023};
static int local_port_range[2] = {DYNAMIC_PORT_MIN, DYNAMIC_PORT_MAX};
static struct ctl_table_header *phonet_table_hrd;

static void set_local_port_range(int range[2])
{
        write_seqlock(&local_port_range_lock);
        local_port_range[0] = range[0];
        local_port_range[1] = range[1];
        write_sequnlock(&local_port_range_lock);
}

void phonet_get_local_port_range(int *min, int *max)
{
        unsigned int seq;

        do {
                seq = read_seqbegin(&local_port_range_lock);
                if (min)
                        *min = local_port_range[0];
                if (max)
                        *max = local_port_range[1];
        } while (read_seqretry(&local_port_range_lock, seq));
}

static int proc_local_port_range(const struct ctl_table *table, int write,
                                 void *buffer, size_t *lenp, loff_t *ppos)
{
        int ret;
        int range[2] = {local_port_range[0], local_port_range[1]};
        struct ctl_table tmp = {
                .data = &range,
                .maxlen = sizeof(range),
                .mode = table->mode,
                .extra1 = &local_port_range_min,
                .extra2 = &local_port_range_max,
        };

        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);

        if (write && ret == 0) {
                if (range[1] < range[0])
                        ret = -EINVAL;
                else
                        set_local_port_range(range);
        }

        return ret;
}

static struct ctl_table phonet_table[] = {
        {
                .procname        = "local_port_range",
                .data                = &local_port_range,
                .maxlen                = sizeof(local_port_range),
                .mode                = 0644,
                .proc_handler        = proc_local_port_range,
        },
};

int __init phonet_sysctl_init(void)
{
        phonet_table_hrd = register_net_sysctl(&init_net, "net/phonet", phonet_table);
        return phonet_table_hrd == NULL ? -ENOMEM : 0;
}

void phonet_sysctl_exit(void)
{
        unregister_net_sysctl_table(phonet_table_hrd);
}











  381 



























































    8 
    8 







    1 




    7 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
// SPDX-License-Identifier: GPL-2.0
#include <net/ip.h>
#include <net/ip6_checksum.h>
#include <net/udp.h>
#include <asm/checksum.h>

#ifndef _HAVE_ARCH_IPV6_CSUM
__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
                        const struct in6_addr *daddr,
                        __u32 len, __u8 proto, __wsum csum)
{

        int carry;
        __u32 ulen;
        __u32 uproto;
        __u32 sum = (__force u32)csum;

        sum += (__force u32)saddr->s6_addr32[0];
        carry = (sum < (__force u32)saddr->s6_addr32[0]);
        sum += carry;

        sum += (__force u32)saddr->s6_addr32[1];
        carry = (sum < (__force u32)saddr->s6_addr32[1]);
        sum += carry;

        sum += (__force u32)saddr->s6_addr32[2];
        carry = (sum < (__force u32)saddr->s6_addr32[2]);
        sum += carry;

        sum += (__force u32)saddr->s6_addr32[3];
        carry = (sum < (__force u32)saddr->s6_addr32[3]);
        sum += carry;

        sum += (__force u32)daddr->s6_addr32[0];
        carry = (sum < (__force u32)daddr->s6_addr32[0]);
        sum += carry;

        sum += (__force u32)daddr->s6_addr32[1];
        carry = (sum < (__force u32)daddr->s6_addr32[1]);
        sum += carry;

        sum += (__force u32)daddr->s6_addr32[2];
        carry = (sum < (__force u32)daddr->s6_addr32[2]);
        sum += carry;

        sum += (__force u32)daddr->s6_addr32[3];
        carry = (sum < (__force u32)daddr->s6_addr32[3]);
        sum += carry;

        ulen = (__force u32)htonl((__u32) len);
        sum += ulen;
        carry = (sum < ulen);
        sum += carry;

        uproto = (__force u32)htonl(proto);
        sum += uproto;
        carry = (sum < uproto);
        sum += carry;

        return csum_fold((__force __wsum)sum);
}
EXPORT_SYMBOL(csum_ipv6_magic);
#endif

/* Function to set UDP checksum for an IPv6 UDP packet. This is intended
 * for the simple case like when setting the checksum for a UDP tunnel.
 */
void udp6_set_csum(bool nocheck, struct sk_buff *skb,
                   const struct in6_addr *saddr,
                   const struct in6_addr *daddr, int len)
{
        struct udphdr *uh = udp_hdr(skb);

        if (nocheck)
                uh->check = 0;
        else if (skb_is_gso(skb))
                uh->check = ~udp_v6_check(len, saddr, daddr, 0);
        else if (skb->ip_summed == CHECKSUM_PARTIAL) {
                uh->check = 0;
                uh->check = udp_v6_check(len, saddr, daddr, lco_csum(skb));
                if (uh->check == 0)
                        uh->check = CSUM_MANGLED_0;
        } else {
                skb->ip_summed = CHECKSUM_PARTIAL;
                skb->csum_start = skb_transport_header(skb) - skb->head;
                skb->csum_offset = offsetof(struct udphdr, check);
                uh->check = ~udp_v6_check(len, saddr, daddr, 0);
        }
}
EXPORT_SYMBOL(udp6_set_csum);















































































































































































































































































































































  761 


  761 










  676 
  761 
  761 







































  672 




  696 





































































































  605 




  605 








































  605 































  605 








  605 









  605 






  605 
  605 
































































  605 










































    1 
    1 
    1 


    1 


















































    1 

    1 

































    1 
    1 





















































































    2 
    1 
    2 







































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_RMAP_H
#define _LINUX_RMAP_H
/*
 * Declarations for Reverse Mapping functions in mm/rmap.c
 */

#include <linux/list.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/rwsem.h>
#include <linux/memcontrol.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/memremap.h>
#include <linux/bit_spinlock.h>

/*
 * The anon_vma heads a list of private "related" vmas, to scan if
 * an anonymous page pointing to this anon_vma needs to be unmapped:
 * the vmas on the list will be related by forking, or by splitting.
 *
 * Since vmas come and go as they are split and merged (particularly
 * in mprotect), the mapping field of an anonymous page cannot point
 * directly to a vma: instead it points to an anon_vma, on whose list
 * the related vmas can be easily linked or unlinked.
 *
 * After unlinking the last vma on the list, we must garbage collect
 * the anon_vma object itself: we're guaranteed no page can be
 * pointing to this anon_vma once its vma list is empty.
 */
struct anon_vma {
        struct anon_vma *root;                /* Root of this anon_vma tree */
        struct rw_semaphore rwsem;        /* W: modification, R: walking the list */
        /*
         * The refcount is taken on an anon_vma when there is no
         * guarantee that the vma of page tables will exist for
         * the duration of the operation. A caller that takes
         * the reference is responsible for clearing up the
         * anon_vma if they are the last user on release
         */
        atomic_t refcount;

        /*
         * Count of child anon_vmas. Equals to the count of all anon_vmas that
         * have ->parent pointing to this one, including itself.
         *
         * This counter is used for making decision about reusing anon_vma
         * instead of forking new one. See comments in function anon_vma_clone.
         */
        unsigned long num_children;
        /* Count of VMAs whose ->anon_vma pointer points to this object. */
        unsigned long num_active_vmas;

        struct anon_vma *parent;        /* Parent of this anon_vma */

        /*
         * NOTE: the LSB of the rb_root.rb_node is set by
         * mm_take_all_locks() _after_ taking the above lock. So the
         * rb_root must only be read/written after taking the above lock
         * to be sure to see a valid next pointer. The LSB bit itself
         * is serialized by a system wide lock only visible to
         * mm_take_all_locks() (mm_all_locks_mutex).
         */

        /* Interval tree of private "related" vmas */
        struct rb_root_cached rb_root;
};

/*
 * The copy-on-write semantics of fork mean that an anon_vma
 * can become associated with multiple processes. Furthermore,
 * each child process will have its own anon_vma, where new
 * pages for that process are instantiated.
 *
 * This structure allows us to find the anon_vmas associated
 * with a VMA, or the VMAs associated with an anon_vma.
 * The "same_vma" list contains the anon_vma_chains linking
 * all the anon_vmas associated with this VMA.
 * The "rb" field indexes on an interval tree the anon_vma_chains
 * which link all the VMAs associated with this anon_vma.
 */
struct anon_vma_chain {
        struct vm_area_struct *vma;
        struct anon_vma *anon_vma;
        struct list_head same_vma;   /* locked by mmap_lock & page_table_lock */
        struct rb_node rb;                        /* locked by anon_vma->rwsem */
        unsigned long rb_subtree_last;
#ifdef CONFIG_DEBUG_VM_RB
        unsigned long cached_vma_start, cached_vma_last;
#endif
};

enum ttu_flags {
        TTU_USE_SHARED_ZEROPAGE        = 0x2,        /* for unused pages of large folios */
        TTU_SPLIT_HUGE_PMD        = 0x4,        /* split huge PMD if any */
        TTU_IGNORE_MLOCK        = 0x8,        /* ignore mlock */
        TTU_SYNC                = 0x10,        /* avoid racy checks with PVMW_SYNC */
        TTU_HWPOISON                = 0x20,        /* do convert pte to hwpoison entry */
        TTU_BATCH_FLUSH                = 0x40,        /* Batch TLB flushes where possible
                                         * and caller guarantees they will
                                         * do a final flush if necessary */
        TTU_RMAP_LOCKED                = 0x80,        /* do not grab rmap lock:
                                         * caller holds it */
};

#ifdef CONFIG_MMU

void anon_vma_init(void);        /* create anon_vma_cachep */

#ifdef CONFIG_MM_ID
static __always_inline void folio_lock_large_mapcount(struct folio *folio)
{
        bit_spin_lock(FOLIO_MM_IDS_LOCK_BITNUM, &folio->_mm_ids);
}

static __always_inline void folio_unlock_large_mapcount(struct folio *folio)
{
        __bit_spin_unlock(FOLIO_MM_IDS_LOCK_BITNUM, &folio->_mm_ids);
}

static inline unsigned int folio_mm_id(const struct folio *folio, int idx)
{
        VM_WARN_ON_ONCE(idx != 0 && idx != 1);
        return folio->_mm_id[idx] & MM_ID_MASK;
}

static inline void folio_set_mm_id(struct folio *folio, int idx, mm_id_t id)
{
        VM_WARN_ON_ONCE(idx != 0 && idx != 1);
        folio->_mm_id[idx] &= ~MM_ID_MASK;
        folio->_mm_id[idx] |= id;
}

static inline void __folio_large_mapcount_sanity_checks(const struct folio *folio,
                int diff, mm_id_t mm_id)
{
        VM_WARN_ON_ONCE(!folio_test_large(folio) || folio_test_hugetlb(folio));
        VM_WARN_ON_ONCE(diff <= 0);
        VM_WARN_ON_ONCE(mm_id < MM_ID_MIN || mm_id > MM_ID_MAX);

        /*
         * Make sure we can detect at least one complete PTE mapping of the
         * folio in a single MM as "exclusively mapped". This is primarily
         * a check on 32bit, where we currently reduce the size of the per-MM
         * mapcount to a short.
         */
        VM_WARN_ON_ONCE(diff > folio_large_nr_pages(folio));
        VM_WARN_ON_ONCE(folio_large_nr_pages(folio) - 1 > MM_ID_MAPCOUNT_MAX);

        VM_WARN_ON_ONCE(folio_mm_id(folio, 0) == MM_ID_DUMMY &&
                        folio->_mm_id_mapcount[0] != -1);
        VM_WARN_ON_ONCE(folio_mm_id(folio, 0) != MM_ID_DUMMY &&
                        folio->_mm_id_mapcount[0] < 0);
        VM_WARN_ON_ONCE(folio_mm_id(folio, 1) == MM_ID_DUMMY &&
                        folio->_mm_id_mapcount[1] != -1);
        VM_WARN_ON_ONCE(folio_mm_id(folio, 1) != MM_ID_DUMMY &&
                        folio->_mm_id_mapcount[1] < 0);
        VM_WARN_ON_ONCE(!folio_mapped(folio) &&
                        test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids));
}

static __always_inline void folio_set_large_mapcount(struct folio *folio,
                int mapcount, struct vm_area_struct *vma)
{
        __folio_large_mapcount_sanity_checks(folio, mapcount, vma->vm_mm->mm_id);

        VM_WARN_ON_ONCE(folio_mm_id(folio, 0) != MM_ID_DUMMY);
        VM_WARN_ON_ONCE(folio_mm_id(folio, 1) != MM_ID_DUMMY);

        /* Note: mapcounts start at -1. */
        atomic_set(&folio->_large_mapcount, mapcount - 1);
        folio->_mm_id_mapcount[0] = mapcount - 1;
        folio_set_mm_id(folio, 0, vma->vm_mm->mm_id);
}

static __always_inline int folio_add_return_large_mapcount(struct folio *folio,
                int diff, struct vm_area_struct *vma)
{
        const mm_id_t mm_id = vma->vm_mm->mm_id;
        int new_mapcount_val;

        folio_lock_large_mapcount(folio);
        __folio_large_mapcount_sanity_checks(folio, diff, mm_id);

        new_mapcount_val = atomic_read(&folio->_large_mapcount) + diff;
        atomic_set(&folio->_large_mapcount, new_mapcount_val);

        /*
         * If a folio is mapped more than once into an MM on 32bit, we
         * can in theory overflow the per-MM mapcount (although only for
         * fairly large folios), turning it negative. In that case, just
         * free up the slot and mark the folio "mapped shared", otherwise
         * we might be in trouble when unmapping pages later.
         */
        if (folio_mm_id(folio, 0) == mm_id) {
                folio->_mm_id_mapcount[0] += diff;
                if (!IS_ENABLED(CONFIG_64BIT) && unlikely(folio->_mm_id_mapcount[0] < 0)) {
                        folio->_mm_id_mapcount[0] = -1;
                        folio_set_mm_id(folio, 0, MM_ID_DUMMY);
                        folio->_mm_ids |= FOLIO_MM_IDS_SHARED_BIT;
                }
        } else if (folio_mm_id(folio, 1) == mm_id) {
                folio->_mm_id_mapcount[1] += diff;
                if (!IS_ENABLED(CONFIG_64BIT) && unlikely(folio->_mm_id_mapcount[1] < 0)) {
                        folio->_mm_id_mapcount[1] = -1;
                        folio_set_mm_id(folio, 1, MM_ID_DUMMY);
                        folio->_mm_ids |= FOLIO_MM_IDS_SHARED_BIT;
                }
        } else if (folio_mm_id(folio, 0) == MM_ID_DUMMY) {
                folio_set_mm_id(folio, 0, mm_id);
                folio->_mm_id_mapcount[0] = diff - 1;
                /* We might have other mappings already. */
                if (new_mapcount_val != diff - 1)
                        folio->_mm_ids |= FOLIO_MM_IDS_SHARED_BIT;
        } else if (folio_mm_id(folio, 1) == MM_ID_DUMMY) {
                folio_set_mm_id(folio, 1, mm_id);
                folio->_mm_id_mapcount[1] = diff - 1;
                /* Slot 0 certainly has mappings as well. */
                folio->_mm_ids |= FOLIO_MM_IDS_SHARED_BIT;
        }
        folio_unlock_large_mapcount(folio);
        return new_mapcount_val + 1;
}
#define folio_add_large_mapcount folio_add_return_large_mapcount

static __always_inline int folio_sub_return_large_mapcount(struct folio *folio,
                int diff, struct vm_area_struct *vma)
{
        const mm_id_t mm_id = vma->vm_mm->mm_id;
        int new_mapcount_val;

        folio_lock_large_mapcount(folio);
        __folio_large_mapcount_sanity_checks(folio, diff, mm_id);

        new_mapcount_val = atomic_read(&folio->_large_mapcount) - diff;
        atomic_set(&folio->_large_mapcount, new_mapcount_val);

        /*
         * There are valid corner cases where we might underflow a per-MM
         * mapcount (some mappings added when no slot was free, some mappings
         * added once a slot was free), so we always set it to -1 once we go
         * negative.
         */
        if (folio_mm_id(folio, 0) == mm_id) {
                folio->_mm_id_mapcount[0] -= diff;
                if (folio->_mm_id_mapcount[0] >= 0)
                        goto out;
                folio->_mm_id_mapcount[0] = -1;
                folio_set_mm_id(folio, 0, MM_ID_DUMMY);
        } else if (folio_mm_id(folio, 1) == mm_id) {
                folio->_mm_id_mapcount[1] -= diff;
                if (folio->_mm_id_mapcount[1] >= 0)
                        goto out;
                folio->_mm_id_mapcount[1] = -1;
                folio_set_mm_id(folio, 1, MM_ID_DUMMY);
        }

        /*
         * If one MM slot owns all mappings, the folio is mapped exclusively.
         * Note that if the folio is now unmapped (new_mapcount_val == -1), both
         * slots must be free (mapcount == -1), and we'll also mark it as
         * exclusive.
         */
        if (folio->_mm_id_mapcount[0] == new_mapcount_val ||
            folio->_mm_id_mapcount[1] == new_mapcount_val)
                folio->_mm_ids &= ~FOLIO_MM_IDS_SHARED_BIT;
out:
        folio_unlock_large_mapcount(folio);
        return new_mapcount_val + 1;
}
#define folio_sub_large_mapcount folio_sub_return_large_mapcount
#else /* !CONFIG_MM_ID */
/*
 * See __folio_rmap_sanity_checks(), we might map large folios even without
 * CONFIG_TRANSPARENT_HUGEPAGE. We'll keep that working for now.
 */
static inline void folio_set_large_mapcount(struct folio *folio, int mapcount,
                struct vm_area_struct *vma)
{
        /* Note: mapcounts start at -1. */
        atomic_set(&folio->_large_mapcount, mapcount - 1);
}

static inline void folio_add_large_mapcount(struct folio *folio,
                int diff, struct vm_area_struct *vma)
{
        atomic_add(diff, &folio->_large_mapcount);
}

static inline int folio_add_return_large_mapcount(struct folio *folio,
                int diff, struct vm_area_struct *vma)
{
        BUILD_BUG();
}

static inline void folio_sub_large_mapcount(struct folio *folio,
                int diff, struct vm_area_struct *vma)
{
        atomic_sub(diff, &folio->_large_mapcount);
}

static inline int folio_sub_return_large_mapcount(struct folio *folio,
                int diff, struct vm_area_struct *vma)
{
        BUILD_BUG();
}
#endif /* CONFIG_MM_ID */

#define folio_inc_large_mapcount(folio, vma) \
        folio_add_large_mapcount(folio, 1, vma)
#define folio_inc_return_large_mapcount(folio, vma) \
        folio_add_return_large_mapcount(folio, 1, vma)
#define folio_dec_large_mapcount(folio, vma) \
        folio_sub_large_mapcount(folio, 1, vma)
#define folio_dec_return_large_mapcount(folio, vma) \
        folio_sub_return_large_mapcount(folio, 1, vma)

/* RMAP flags, currently only relevant for some anon rmap operations. */
typedef int __bitwise rmap_t;

/*
 * No special request: A mapped anonymous (sub)page is possibly shared between
 * processes.
 */
#define RMAP_NONE                ((__force rmap_t)0)

/* The anonymous (sub)page is exclusive to a single process. */
#define RMAP_EXCLUSIVE                ((__force rmap_t)BIT(0))

static __always_inline void __folio_rmap_sanity_checks(const struct folio *folio,
                const struct page *page, int nr_pages, enum pgtable_level level)
{
        /* hugetlb folios are handled separately. */
        VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio);

        /* When (un)mapping zeropages, we should never touch ref+mapcount. */
        VM_WARN_ON_FOLIO(is_zero_folio(folio), folio);

        /*
         * TODO: we get driver-allocated folios that have nothing to do with
         * the rmap using vm_insert_page(); therefore, we cannot assume that
         * folio_test_large_rmappable() holds for large folios. We should
         * handle any desired mapcount+stats accounting for these folios in
         * VM_MIXEDMAP VMAs separately, and then sanity-check here that
         * we really only get rmappable folios.
         */

        VM_WARN_ON_ONCE(nr_pages <= 0);
        VM_WARN_ON_FOLIO(page_folio(page) != folio, folio);
        VM_WARN_ON_FOLIO(page_folio(page + nr_pages - 1) != folio, folio);

        switch (level) {
        case PGTABLE_LEVEL_PTE:
                break;
        case PGTABLE_LEVEL_PMD:
                /*
                 * We don't support folios larger than a single PMD yet. So
                 * when PGTABLE_LEVEL_PMD is set, we assume that we are creating
                 * a single "entire" mapping of the folio.
                 */
                VM_WARN_ON_FOLIO(folio_nr_pages(folio) != HPAGE_PMD_NR, folio);
                VM_WARN_ON_FOLIO(nr_pages != HPAGE_PMD_NR, folio);
                break;
        case PGTABLE_LEVEL_PUD:
                /*
                 * Assume that we are creating a single "entire" mapping of the
                 * folio.
                 */
                VM_WARN_ON_FOLIO(folio_nr_pages(folio) != HPAGE_PUD_NR, folio);
                VM_WARN_ON_FOLIO(nr_pages != HPAGE_PUD_NR, folio);
                break;
        default:
                BUILD_BUG();
        }

        /*
         * Anon folios must have an associated live anon_vma as long as they're
         * mapped into userspace.
         * Note that the atomic_read() mainly does two things:
         *
         * 1. In KASAN builds with CONFIG_SLUB_RCU_DEBUG, it causes KASAN to
         *    check that the associated anon_vma has not yet been freed (subject
         *    to KASAN's usual limitations). This check will pass if the
         *    anon_vma's refcount has already dropped to 0 but an RCU grace
         *    period hasn't passed since then.
         * 2. If the anon_vma has not yet been freed, it checks that the
         *    anon_vma still has a nonzero refcount (as opposed to being in the
         *    middle of an RCU delay for getting freed).
         */
        if (folio_test_anon(folio) && !folio_test_ksm(folio)) {
                unsigned long mapping = (unsigned long)folio->mapping;
                struct anon_vma *anon_vma;

                anon_vma = (void *)(mapping - FOLIO_MAPPING_ANON);
                VM_WARN_ON_FOLIO(atomic_read(&anon_vma->refcount) == 0, folio);
        }
}

/*
 * rmap interfaces called when adding or removing pte of page
 */
void folio_move_anon_rmap(struct folio *, struct vm_area_struct *);
void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages,
                struct vm_area_struct *, unsigned long address, rmap_t flags);
#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \
        folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags)
void folio_add_anon_rmap_pmd(struct folio *, struct page *,
                struct vm_area_struct *, unsigned long address, rmap_t flags);
void folio_add_new_anon_rmap(struct folio *, struct vm_area_struct *,
                unsigned long address, rmap_t flags);
void folio_add_file_rmap_ptes(struct folio *, struct page *, int nr_pages,
                struct vm_area_struct *);
#define folio_add_file_rmap_pte(folio, page, vma) \
        folio_add_file_rmap_ptes(folio, page, 1, vma)
void folio_add_file_rmap_pmd(struct folio *, struct page *,
                struct vm_area_struct *);
void folio_add_file_rmap_pud(struct folio *, struct page *,
                struct vm_area_struct *);
void folio_remove_rmap_ptes(struct folio *, struct page *, int nr_pages,
                struct vm_area_struct *);
#define folio_remove_rmap_pte(folio, page, vma) \
        folio_remove_rmap_ptes(folio, page, 1, vma)
void folio_remove_rmap_pmd(struct folio *, struct page *,
                struct vm_area_struct *);
void folio_remove_rmap_pud(struct folio *, struct page *,
                struct vm_area_struct *);

void hugetlb_add_anon_rmap(struct folio *, struct vm_area_struct *,
                unsigned long address, rmap_t flags);
void hugetlb_add_new_anon_rmap(struct folio *, struct vm_area_struct *,
                unsigned long address);

/* See folio_try_dup_anon_rmap_*() */
static inline int hugetlb_try_dup_anon_rmap(struct folio *folio,
                struct vm_area_struct *vma)
{
        VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
        VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);

        if (PageAnonExclusive(&folio->page)) {
                if (unlikely(folio_needs_cow_for_dma(vma, folio)))
                        return -EBUSY;
                ClearPageAnonExclusive(&folio->page);
        }
        atomic_inc(&folio->_entire_mapcount);
        atomic_inc(&folio->_large_mapcount);
        return 0;
}

/* See folio_try_share_anon_rmap_*() */
static inline int hugetlb_try_share_anon_rmap(struct folio *folio)
{
        VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
        VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
        VM_WARN_ON_FOLIO(!PageAnonExclusive(&folio->page), folio);

        /* Paired with the memory barrier in try_grab_folio(). */
        if (IS_ENABLED(CONFIG_HAVE_GUP_FAST))
                smp_mb();

        if (unlikely(folio_maybe_dma_pinned(folio)))
                return -EBUSY;
        ClearPageAnonExclusive(&folio->page);

        /*
         * This is conceptually a smp_wmb() paired with the smp_rmb() in
         * gup_must_unshare().
         */
        if (IS_ENABLED(CONFIG_HAVE_GUP_FAST))
                smp_mb__after_atomic();
        return 0;
}

static inline void hugetlb_add_file_rmap(struct folio *folio)
{
        VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);
        VM_WARN_ON_FOLIO(folio_test_anon(folio), folio);

        atomic_inc(&folio->_entire_mapcount);
        atomic_inc(&folio->_large_mapcount);
}

static inline void hugetlb_remove_rmap(struct folio *folio)
{
        VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio);

        atomic_dec(&folio->_entire_mapcount);
        atomic_dec(&folio->_large_mapcount);
}

static __always_inline void __folio_dup_file_rmap(struct folio *folio,
                struct page *page, int nr_pages, struct vm_area_struct *dst_vma,
                enum pgtable_level level)
{
        const int orig_nr_pages = nr_pages;

        __folio_rmap_sanity_checks(folio, page, nr_pages, level);

        switch (level) {
        case PGTABLE_LEVEL_PTE:
                if (!folio_test_large(folio)) {
                        atomic_inc(&folio->_mapcount);
                        break;
                }

                if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) {
                        do {
                                atomic_inc(&page->_mapcount);
                        } while (page++, --nr_pages > 0);
                }
                folio_add_large_mapcount(folio, orig_nr_pages, dst_vma);
                break;
        case PGTABLE_LEVEL_PMD:
        case PGTABLE_LEVEL_PUD:
                atomic_inc(&folio->_entire_mapcount);
                folio_inc_large_mapcount(folio, dst_vma);
                break;
        default:
                BUILD_BUG();
        }
}

/**
 * folio_dup_file_rmap_ptes - duplicate PTE mappings of a page range of a folio
 * @folio:        The folio to duplicate the mappings of
 * @page:        The first page to duplicate the mappings of
 * @nr_pages:        The number of pages of which the mapping will be duplicated
 * @dst_vma:        The destination vm area
 *
 * The page range of the folio is defined by [page, page + nr_pages)
 *
 * The caller needs to hold the page table lock.
 */
static inline void folio_dup_file_rmap_ptes(struct folio *folio,
                struct page *page, int nr_pages, struct vm_area_struct *dst_vma)
{
        __folio_dup_file_rmap(folio, page, nr_pages, dst_vma, PGTABLE_LEVEL_PTE);
}

static __always_inline void folio_dup_file_rmap_pte(struct folio *folio,
                struct page *page, struct vm_area_struct *dst_vma)
{
        __folio_dup_file_rmap(folio, page, 1, dst_vma, PGTABLE_LEVEL_PTE);
}

/**
 * folio_dup_file_rmap_pmd - duplicate a PMD mapping of a page range of a folio
 * @folio:        The folio to duplicate the mapping of
 * @page:        The first page to duplicate the mapping of
 * @dst_vma:        The destination vm area
 *
 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
 *
 * The caller needs to hold the page table lock.
 */
static inline void folio_dup_file_rmap_pmd(struct folio *folio,
                struct page *page, struct vm_area_struct *dst_vma)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
        __folio_dup_file_rmap(folio, page, HPAGE_PMD_NR, dst_vma, PGTABLE_LEVEL_PTE);
#else
        WARN_ON_ONCE(true);
#endif
}

static __always_inline int __folio_try_dup_anon_rmap(struct folio *folio,
                struct page *page, int nr_pages, struct vm_area_struct *dst_vma,
                struct vm_area_struct *src_vma, enum pgtable_level level)
{
        const int orig_nr_pages = nr_pages;
        bool maybe_pinned;
        int i;

        VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
        __folio_rmap_sanity_checks(folio, page, nr_pages, level);

        /*
         * If this folio may have been pinned by the parent process,
         * don't allow to duplicate the mappings but instead require to e.g.,
         * copy the subpage immediately for the child so that we'll always
         * guarantee the pinned folio won't be randomly replaced in the
         * future on write faults.
         */
        maybe_pinned = likely(!folio_is_device_private(folio)) &&
                       unlikely(folio_needs_cow_for_dma(src_vma, folio));

        /*
         * No need to check+clear for already shared PTEs/PMDs of the
         * folio. But if any page is PageAnonExclusive, we must fallback to
         * copying if the folio maybe pinned.
         */
        switch (level) {
        case PGTABLE_LEVEL_PTE:
                if (unlikely(maybe_pinned)) {
                        for (i = 0; i < nr_pages; i++)
                                if (PageAnonExclusive(page + i))
                                        return -EBUSY;
                }

                if (!folio_test_large(folio)) {
                        if (PageAnonExclusive(page))
                                ClearPageAnonExclusive(page);
                        atomic_inc(&folio->_mapcount);
                        break;
                }

                do {
                        if (PageAnonExclusive(page))
                                ClearPageAnonExclusive(page);
                        if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
                                atomic_inc(&page->_mapcount);
                } while (page++, --nr_pages > 0);
                folio_add_large_mapcount(folio, orig_nr_pages, dst_vma);
                break;
        case PGTABLE_LEVEL_PMD:
        case PGTABLE_LEVEL_PUD:
                if (PageAnonExclusive(page)) {
                        if (unlikely(maybe_pinned))
                                return -EBUSY;
                        ClearPageAnonExclusive(page);
                }
                atomic_inc(&folio->_entire_mapcount);
                folio_inc_large_mapcount(folio, dst_vma);
                break;
        default:
                BUILD_BUG();
        }
        return 0;
}

/**
 * folio_try_dup_anon_rmap_ptes - try duplicating PTE mappings of a page range
 *                                  of a folio
 * @folio:        The folio to duplicate the mappings of
 * @page:        The first page to duplicate the mappings of
 * @nr_pages:        The number of pages of which the mapping will be duplicated
 * @dst_vma:        The destination vm area
 * @src_vma:        The vm area from which the mappings are duplicated
 *
 * The page range of the folio is defined by [page, page + nr_pages)
 *
 * The caller needs to hold the page table lock and the
 * vma->vma_mm->write_protect_seq.
 *
 * Duplicating the mappings can only fail if the folio may be pinned; device
 * private folios cannot get pinned and consequently this function cannot fail
 * for them.
 *
 * If duplicating the mappings succeeded, the duplicated PTEs have to be R/O in
 * the parent and the child. They must *not* be writable after this call
 * succeeded.
 *
 * Returns 0 if duplicating the mappings succeeded. Returns -EBUSY otherwise.
 */
static inline int folio_try_dup_anon_rmap_ptes(struct folio *folio,
                struct page *page, int nr_pages, struct vm_area_struct *dst_vma,
                struct vm_area_struct *src_vma)
{
        return __folio_try_dup_anon_rmap(folio, page, nr_pages, dst_vma,
                                         src_vma, PGTABLE_LEVEL_PTE);
}

static __always_inline int folio_try_dup_anon_rmap_pte(struct folio *folio,
                struct page *page, struct vm_area_struct *dst_vma,
                struct vm_area_struct *src_vma)
{
        return __folio_try_dup_anon_rmap(folio, page, 1, dst_vma, src_vma,
                                         PGTABLE_LEVEL_PTE);
}

/**
 * folio_try_dup_anon_rmap_pmd - try duplicating a PMD mapping of a page range
 *                                 of a folio
 * @folio:        The folio to duplicate the mapping of
 * @page:        The first page to duplicate the mapping of
 * @dst_vma:        The destination vm area
 * @src_vma:        The vm area from which the mapping is duplicated
 *
 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
 *
 * The caller needs to hold the page table lock and the
 * vma->vma_mm->write_protect_seq.
 *
 * Duplicating the mapping can only fail if the folio may be pinned; device
 * private folios cannot get pinned and consequently this function cannot fail
 * for them.
 *
 * If duplicating the mapping succeeds, the duplicated PMD has to be R/O in
 * the parent and the child. They must *not* be writable after this call
 * succeeded.
 *
 * Returns 0 if duplicating the mapping succeeded. Returns -EBUSY otherwise.
 */
static inline int folio_try_dup_anon_rmap_pmd(struct folio *folio,
                struct page *page, struct vm_area_struct *dst_vma,
                struct vm_area_struct *src_vma)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
        return __folio_try_dup_anon_rmap(folio, page, HPAGE_PMD_NR, dst_vma,
                                         src_vma, PGTABLE_LEVEL_PMD);
#else
        WARN_ON_ONCE(true);
        return -EBUSY;
#endif
}

static __always_inline int __folio_try_share_anon_rmap(struct folio *folio,
                struct page *page, int nr_pages, enum pgtable_level level)
{
        VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
        VM_WARN_ON_FOLIO(!PageAnonExclusive(page), folio);
        __folio_rmap_sanity_checks(folio, page, nr_pages, level);

        /* device private folios cannot get pinned via GUP. */
        if (unlikely(folio_is_device_private(folio))) {
                ClearPageAnonExclusive(page);
                return 0;
        }

        /*
         * We have to make sure that when we clear PageAnonExclusive, that
         * the page is not pinned and that concurrent GUP-fast won't succeed in
         * concurrently pinning the page.
         *
         * Conceptually, PageAnonExclusive clearing consists of:
         * (A1) Clear PTE
         * (A2) Check if the page is pinned; back off if so.
         * (A3) Clear PageAnonExclusive
         * (A4) Restore PTE (optional, but certainly not writable)
         *
         * When clearing PageAnonExclusive, we cannot possibly map the page
         * writable again, because anon pages that may be shared must never
         * be writable. So in any case, if the PTE was writable it cannot
         * be writable anymore afterwards and there would be a PTE change. Only
         * if the PTE wasn't writable, there might not be a PTE change.
         *
         * Conceptually, GUP-fast pinning of an anon page consists of:
         * (B1) Read the PTE
         * (B2) FOLL_WRITE: check if the PTE is not writable; back off if so.
         * (B3) Pin the mapped page
         * (B4) Check if the PTE changed by re-reading it; back off if so.
         * (B5) If the original PTE is not writable, check if
         *        PageAnonExclusive is not set; back off if so.
         *
         * If the PTE was writable, we only have to make sure that GUP-fast
         * observes a PTE change and properly backs off.
         *
         * If the PTE was not writable, we have to make sure that GUP-fast either
         * detects a (temporary) PTE change or that PageAnonExclusive is cleared
         * and properly backs off.
         *
         * Consequently, when clearing PageAnonExclusive(), we have to make
         * sure that (A1), (A2)/(A3) and (A4) happen in the right memory
         * order. In GUP-fast pinning code, we have to make sure that (B3),(B4)
         * and (B5) happen in the right memory order.
         *
         * We assume that there might not be a memory barrier after
         * clearing/invalidating the PTE (A1) and before restoring the PTE (A4),
         * so we use explicit ones here.
         */

        /* Paired with the memory barrier in try_grab_folio(). */
        if (IS_ENABLED(CONFIG_HAVE_GUP_FAST))
                smp_mb();

        if (unlikely(folio_maybe_dma_pinned(folio)))
                return -EBUSY;
        ClearPageAnonExclusive(page);

        /*
         * This is conceptually a smp_wmb() paired with the smp_rmb() in
         * gup_must_unshare().
         */
        if (IS_ENABLED(CONFIG_HAVE_GUP_FAST))
                smp_mb__after_atomic();
        return 0;
}

/**
 * folio_try_share_anon_rmap_pte - try marking an exclusive anonymous page
 *                                   mapped by a PTE possibly shared to prepare
 *                                   for KSM or temporary unmapping
 * @folio:        The folio to share a mapping of
 * @page:        The mapped exclusive page
 *
 * The caller needs to hold the page table lock and has to have the page table
 * entries cleared/invalidated.
 *
 * This is similar to folio_try_dup_anon_rmap_pte(), however, not used during
 * fork() to duplicate mappings, but instead to prepare for KSM or temporarily
 * unmapping parts of a folio (swap, migration) via folio_remove_rmap_pte().
 *
 * Marking the mapped page shared can only fail if the folio maybe pinned;
 * device private folios cannot get pinned and consequently this function cannot
 * fail.
 *
 * Returns 0 if marking the mapped page possibly shared succeeded. Returns
 * -EBUSY otherwise.
 */
static inline int folio_try_share_anon_rmap_pte(struct folio *folio,
                struct page *page)
{
        return __folio_try_share_anon_rmap(folio, page, 1, PGTABLE_LEVEL_PTE);
}

/**
 * folio_try_share_anon_rmap_pmd - try marking an exclusive anonymous page
 *                                   range mapped by a PMD possibly shared to
 *                                   prepare for temporary unmapping
 * @folio:        The folio to share the mapping of
 * @page:        The first page to share the mapping of
 *
 * The page range of the folio is defined by [page, page + HPAGE_PMD_NR)
 *
 * The caller needs to hold the page table lock and has to have the page table
 * entries cleared/invalidated.
 *
 * This is similar to folio_try_dup_anon_rmap_pmd(), however, not used during
 * fork() to duplicate a mapping, but instead to prepare for temporarily
 * unmapping parts of a folio (swap, migration) via folio_remove_rmap_pmd().
 *
 * Marking the mapped pages shared can only fail if the folio maybe pinned;
 * device private folios cannot get pinned and consequently this function cannot
 * fail.
 *
 * Returns 0 if marking the mapped pages possibly shared succeeded. Returns
 * -EBUSY otherwise.
 */
static inline int folio_try_share_anon_rmap_pmd(struct folio *folio,
                struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
        return __folio_try_share_anon_rmap(folio, page, HPAGE_PMD_NR,
                                           PGTABLE_LEVEL_PMD);
#else
        WARN_ON_ONCE(true);
        return -EBUSY;
#endif
}

/*
 * Called from mm/vmscan.c to handle paging out
 */
int folio_referenced(struct folio *, int is_locked,
                        struct mem_cgroup *memcg, vm_flags_t *vm_flags);

void try_to_migrate(struct folio *folio, enum ttu_flags flags);
void try_to_unmap(struct folio *, enum ttu_flags flags);

struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr,
                void *owner, struct folio **foliop);

/* Avoid racy checks */
#define PVMW_SYNC                (1 << 0)
/* Look for migration entries rather than present PTEs */
#define PVMW_MIGRATION                (1 << 1)

/* Result flags */

/* The page is mapped across page table boundary */
#define PVMW_PGTABLE_CROSSED        (1 << 16)

struct page_vma_mapped_walk {
        unsigned long pfn;
        unsigned long nr_pages;
        pgoff_t pgoff;
        struct vm_area_struct *vma;
        unsigned long address;
        pmd_t *pmd;
        pte_t *pte;
        spinlock_t *ptl;
        unsigned int flags;
};

#define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags)        \
        struct page_vma_mapped_walk name = {                                \
                .pfn = folio_pfn(_folio),                                \
                .nr_pages = folio_nr_pages(_folio),                        \
                .pgoff = folio_pgoff(_folio),                                \
                .vma = _vma,                                                \
                .address = _address,                                        \
                .flags = _flags,                                        \
        }

static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw)
{
        /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */
        if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma))
                pte_unmap(pvmw->pte);
        if (pvmw->ptl)
                spin_unlock(pvmw->ptl);
}

/**
 * page_vma_mapped_walk_restart - Restart the page table walk.
 * @pvmw: Pointer to struct page_vma_mapped_walk.
 *
 * It restarts the page table walk when changes occur in the page
 * table, such as splitting a PMD. Ensures that the PTL held during
 * the previous walk is released and resets the state to allow for
 * a new walk starting at the current address stored in pvmw->address.
 */
static inline void
page_vma_mapped_walk_restart(struct page_vma_mapped_walk *pvmw)
{
        WARN_ON_ONCE(!pvmw->pmd && !pvmw->pte);

        if (likely(pvmw->ptl))
                spin_unlock(pvmw->ptl);
        else
                WARN_ON_ONCE(1);

        pvmw->ptl = NULL;
        pvmw->pmd = NULL;
        pvmw->pte = NULL;
}

bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw);
unsigned long page_address_in_vma(const struct folio *folio,
                const struct page *, const struct vm_area_struct *);

/*
 * Cleans the PTEs of shared mappings.
 * (and since clean PTEs should also be readonly, write protects them too)
 *
 * returns the number of cleaned PTEs.
 */
int folio_mkclean(struct folio *);

int mapping_wrprotect_range(struct address_space *mapping, pgoff_t pgoff,
                unsigned long pfn, unsigned long nr_pages);

int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff,
                      struct vm_area_struct *vma);

void remove_migration_ptes(struct folio *src, struct folio *dst,
                enum ttu_flags flags);

/*
 * rmap_walk_control: To control rmap traversing for specific needs
 *
 * arg: passed to rmap_one() and invalid_vma()
 * try_lock: bail out if the rmap lock is contended
 * contended: indicate the rmap traversal bailed out due to lock contention
 * rmap_one: executed on each vma where page is mapped
 * done: for checking traversing termination condition
 * anon_lock: for getting anon_lock by optimized way rather than default
 * invalid_vma: for skipping uninterested vma
 */
struct rmap_walk_control {
        void *arg;
        bool try_lock;
        bool contended;
        /*
         * Return false if page table scanning in rmap_walk should be stopped.
         * Otherwise, return true.
         */
        bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma,
                                        unsigned long addr, void *arg);
        int (*done)(struct folio *folio);
        struct anon_vma *(*anon_lock)(const struct folio *folio,
                                      struct rmap_walk_control *rwc);
        bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);
};

void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc);
void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc);
struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
                                          struct rmap_walk_control *rwc);

#else        /* !CONFIG_MMU */

#define anon_vma_init()                do {} while (0)
#define anon_vma_prepare(vma)        (0)

static inline int folio_referenced(struct folio *folio, int is_locked,
                                  struct mem_cgroup *memcg,
                                  vm_flags_t *vm_flags)
{
        *vm_flags = 0;
        return 0;
}

static inline void try_to_unmap(struct folio *folio, enum ttu_flags flags)
{
}

static inline int folio_mkclean(struct folio *folio)
{
        return 0;
}
#endif        /* CONFIG_MMU */

#endif        /* _LINUX_RMAP_H */

































































































































































































































    7 
    7 






































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2018 Oracle and/or its affiliates. All rights reserved. */

#include <crypto/aead.h>
#include <linux/debugfs.h>
#include <net/xfrm.h>

#include "netdevsim.h"

#define NSIM_IPSEC_AUTH_BITS        128

static ssize_t nsim_dbg_netdev_ops_read(struct file *filp,
                                        char __user *buffer,
                                        size_t count, loff_t *ppos)
{
        struct netdevsim *ns = filp->private_data;
        struct nsim_ipsec *ipsec = &ns->ipsec;
        size_t bufsize;
        char *buf, *p;
        int len;
        int i;

        /* the buffer needed is
         * (num SAs * 3 lines each * ~60 bytes per line) + one more line
         */
        bufsize = (ipsec->count * 4 * 60) + 60;
        buf = kzalloc(bufsize, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;

        p = buf;
        p += scnprintf(p, bufsize - (p - buf),
                       "SA count=%u tx=%u\n",
                       ipsec->count, ipsec->tx);

        for (i = 0; i < NSIM_IPSEC_MAX_SA_COUNT; i++) {
                struct nsim_sa *sap = &ipsec->sa[i];

                if (!sap->used)
                        continue;

                if (sap->xs->props.family == AF_INET6)
                        p += scnprintf(p, bufsize - (p - buf),
                                       "sa[%i] %cx ipaddr=%pI6c\n",
                                       i, (sap->rx ? 'r' : 't'), &sap->ipaddr);
                else
                        p += scnprintf(p, bufsize - (p - buf),
                                       "sa[%i] %cx ipaddr=%pI4\n",
                                       i, (sap->rx ? 'r' : 't'), &sap->ipaddr[3]);
                p += scnprintf(p, bufsize - (p - buf),
                               "sa[%i]    spi=0x%08x proto=0x%x salt=0x%08x crypt=%d\n",
                               i, be32_to_cpu(sap->xs->id.spi),
                               sap->xs->id.proto, sap->salt, sap->crypt);
                p += scnprintf(p, bufsize - (p - buf),
                               "sa[%i]    key=0x%08x %08x %08x %08x\n",
                               i, sap->key[0], sap->key[1],
                               sap->key[2], sap->key[3]);
        }

        len = simple_read_from_buffer(buffer, count, ppos, buf, p - buf);

        kfree(buf);
        return len;
}

static const struct file_operations ipsec_dbg_fops = {
        .owner = THIS_MODULE,
        .open = simple_open,
        .read = nsim_dbg_netdev_ops_read,
};

static int nsim_ipsec_find_empty_idx(struct nsim_ipsec *ipsec)
{
        u32 i;

        if (ipsec->count == NSIM_IPSEC_MAX_SA_COUNT)
                return -ENOSPC;

        /* search sa table */
        for (i = 0; i < NSIM_IPSEC_MAX_SA_COUNT; i++) {
                if (!ipsec->sa[i].used)
                        return i;
        }

        return -ENOSPC;
}

static int nsim_ipsec_parse_proto_keys(struct net_device *dev,
                                       struct xfrm_state *xs,
                                       u32 *mykey, u32 *mysalt)
{
        const char aes_gcm_name[] = "rfc4106(gcm(aes))";
        unsigned char *key_data;
        char *alg_name = NULL;
        int key_len;

        if (!xs->aead) {
                netdev_err(dev, "Unsupported IPsec algorithm\n");
                return -EINVAL;
        }

        if (xs->aead->alg_icv_len != NSIM_IPSEC_AUTH_BITS) {
                netdev_err(dev, "IPsec offload requires %d bit authentication\n",
                           NSIM_IPSEC_AUTH_BITS);
                return -EINVAL;
        }

        key_data = &xs->aead->alg_key[0];
        key_len = xs->aead->alg_key_len;
        alg_name = xs->aead->alg_name;

        if (strcmp(alg_name, aes_gcm_name)) {
                netdev_err(dev, "Unsupported IPsec algorithm - please use %s\n",
                           aes_gcm_name);
                return -EINVAL;
        }

        /* 160 accounts for 16 byte key and 4 byte salt */
        if (key_len > NSIM_IPSEC_AUTH_BITS) {
                *mysalt = ((u32 *)key_data)[4];
        } else if (key_len == NSIM_IPSEC_AUTH_BITS) {
                *mysalt = 0;
        } else {
                netdev_err(dev, "IPsec hw offload only supports 128 bit keys with optional 32 bit salt\n");
                return -EINVAL;
        }
        memcpy(mykey, key_data, 16);

        return 0;
}

static int nsim_ipsec_add_sa(struct net_device *dev,
                             struct xfrm_state *xs,
                             struct netlink_ext_ack *extack)
{
        struct nsim_ipsec *ipsec;
        struct netdevsim *ns;
        struct nsim_sa sa;
        u16 sa_idx;
        int ret;

        ns = netdev_priv(dev);
        ipsec = &ns->ipsec;

        if (xs->id.proto != IPPROTO_ESP && xs->id.proto != IPPROTO_AH) {
                NL_SET_ERR_MSG_MOD(extack, "Unsupported protocol for ipsec offload");
                return -EINVAL;
        }

        if (xs->calg) {
                NL_SET_ERR_MSG_MOD(extack, "Compression offload not supported");
                return -EINVAL;
        }

        if (xs->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
                NL_SET_ERR_MSG_MOD(extack, "Unsupported ipsec offload type");
                return -EINVAL;
        }

        /* find the first unused index */
        ret = nsim_ipsec_find_empty_idx(ipsec);
        if (ret < 0) {
                NL_SET_ERR_MSG_MOD(extack, "No space for SA in Rx table!");
                return ret;
        }
        sa_idx = (u16)ret;

        memset(&sa, 0, sizeof(sa));
        sa.used = true;
        sa.xs = xs;

        if (sa.xs->id.proto & IPPROTO_ESP)
                sa.crypt = xs->ealg || xs->aead;

        /* get the key and salt */
        ret = nsim_ipsec_parse_proto_keys(dev, xs, sa.key, &sa.salt);
        if (ret) {
                NL_SET_ERR_MSG_MOD(extack, "Failed to get key data for SA table");
                return ret;
        }

        if (xs->xso.dir == XFRM_DEV_OFFLOAD_IN)
                sa.rx = true;

        if (xs->props.family == AF_INET6)
                memcpy(sa.ipaddr, &xs->id.daddr.a6, 16);
        else
                memcpy(&sa.ipaddr[3], &xs->id.daddr.a4, 4);

        /* the preparations worked, so save the info */
        memcpy(&ipsec->sa[sa_idx], &sa, sizeof(sa));

        /* the XFRM stack doesn't like offload_handle == 0,
         * so add a bitflag in case our array index is 0
         */
        xs->xso.offload_handle = sa_idx | NSIM_IPSEC_VALID;
        ipsec->count++;

        return 0;
}

static void nsim_ipsec_del_sa(struct net_device *dev, struct xfrm_state *xs)
{
        struct netdevsim *ns = netdev_priv(dev);
        struct nsim_ipsec *ipsec = &ns->ipsec;
        u16 sa_idx;

        sa_idx = xs->xso.offload_handle & ~NSIM_IPSEC_VALID;
        if (!ipsec->sa[sa_idx].used) {
                netdev_err(ns->netdev, "Invalid SA for delete sa_idx=%d\n",
                           sa_idx);
                return;
        }

        memset(&ipsec->sa[sa_idx], 0, sizeof(struct nsim_sa));
        ipsec->count--;
}

static const struct xfrmdev_ops nsim_xfrmdev_ops = {
        .xdo_dev_state_add        = nsim_ipsec_add_sa,
        .xdo_dev_state_delete        = nsim_ipsec_del_sa,
};

bool nsim_ipsec_tx(struct netdevsim *ns, struct sk_buff *skb)
{
        struct sec_path *sp = skb_sec_path(skb);
        struct nsim_ipsec *ipsec = &ns->ipsec;
        struct xfrm_state *xs;
        struct nsim_sa *tsa;
        u32 sa_idx;

        /* do we even need to check this packet? */
        if (!sp)
                return true;

        if (unlikely(!sp->len)) {
                netdev_err(ns->netdev, "no xfrm state len = %d\n",
                           sp->len);
                return false;
        }

        xs = xfrm_input_state(skb);
        if (unlikely(!xs)) {
                netdev_err(ns->netdev, "no xfrm_input_state() xs = %p\n", xs);
                return false;
        }

        sa_idx = xs->xso.offload_handle & ~NSIM_IPSEC_VALID;
        if (unlikely(sa_idx >= NSIM_IPSEC_MAX_SA_COUNT)) {
                netdev_err(ns->netdev, "bad sa_idx=%d max=%d\n",
                           sa_idx, NSIM_IPSEC_MAX_SA_COUNT);
                return false;
        }

        tsa = &ipsec->sa[sa_idx];
        if (unlikely(!tsa->used)) {
                netdev_err(ns->netdev, "unused sa_idx=%d\n", sa_idx);
                return false;
        }

        if (xs->id.proto != IPPROTO_ESP && xs->id.proto != IPPROTO_AH) {
                netdev_err(ns->netdev, "unexpected proto=%d\n", xs->id.proto);
                return false;
        }

        ipsec->tx++;

        return true;
}

void nsim_ipsec_init(struct netdevsim *ns)
{
        ns->netdev->xfrmdev_ops = &nsim_xfrmdev_ops;

#define NSIM_ESP_FEATURES        (NETIF_F_HW_ESP | \
                                 NETIF_F_HW_ESP_TX_CSUM | \
                                 NETIF_F_GSO_ESP)

        ns->netdev->features |= NSIM_ESP_FEATURES;
        ns->netdev->hw_features |= NSIM_ESP_FEATURES;
        ns->netdev->hw_enc_features |= NSIM_ESP_FEATURES;

        ns->ipsec.pfile = debugfs_create_file("ipsec", 0400,
                                              ns->nsim_dev_port->ddir, ns,
                                              &ipsec_dbg_fops);
}

void nsim_ipsec_teardown(struct netdevsim *ns)
{
        struct nsim_ipsec *ipsec = &ns->ipsec;

        if (ipsec->count)
                netdev_err(ns->netdev, "tearing down IPsec offload with %d SAs left\n",
                           ipsec->count);
        debugfs_remove_recursive(ipsec->pfile);
}































































































































   68 
   68 













   66 






























   68 


   68 


















   79 



    2 


   72 
    5 




    4 
    4 










   66 


















    8 





    8 



    8 




    1 





    1 




    1 





    1 


















































    3 



   55 






   24 














































































































   74 



















  168 




    7 



    6 




  155 






  155 


   11 





   74 
   74 








  150 

    3 




























  153 




   85 



    8 




































   32 

  126 



   28 

   28 










   35 

   35 










  134 





   34 




   52 



  100 












  192 





   58 

   58 

   24 

   23 

   25 

    5 































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
// SPDX-License-Identifier: GPL-2.0
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/err.h>
#include <linux/irq_work.h>
#include <linux/slab.h>
#include <linux/filter.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <linux/wait.h>
#include <linux/poll.h>
#include <linux/kmemleak.h>
#include <uapi/linux/btf.h>
#include <linux/btf_ids.h>
#include <asm/rqspinlock.h>

#define RINGBUF_CREATE_FLAG_MASK (BPF_F_NUMA_NODE | BPF_F_RB_OVERWRITE)

/* non-mmap()'able part of bpf_ringbuf (everything up to consumer page) */
#define RINGBUF_PGOFF \
        (offsetof(struct bpf_ringbuf, consumer_pos) >> PAGE_SHIFT)
/* consumer page and producer page */
#define RINGBUF_POS_PAGES 2
#define RINGBUF_NR_META_PAGES (RINGBUF_PGOFF + RINGBUF_POS_PAGES)

#define RINGBUF_MAX_RECORD_SZ (UINT_MAX/4)

struct bpf_ringbuf {
        wait_queue_head_t waitq;
        struct irq_work work;
        u64 mask;
        struct page **pages;
        int nr_pages;
        bool overwrite_mode;
        rqspinlock_t spinlock ____cacheline_aligned_in_smp;
        /* For user-space producer ring buffers, an atomic_t busy bit is used
         * to synchronize access to the ring buffers in the kernel, rather than
         * the spinlock that is used for kernel-producer ring buffers. This is
         * done because the ring buffer must hold a lock across a BPF program's
         * callback:
         *
         *    __bpf_user_ringbuf_peek() // lock acquired
         * -> program callback_fn()
         * -> __bpf_user_ringbuf_sample_release() // lock released
         *
         * It is unsafe and incorrect to hold an IRQ spinlock across what could
         * be a long execution window, so we instead simply disallow concurrent
         * access to the ring buffer by kernel consumers, and return -EBUSY from
         * __bpf_user_ringbuf_peek() if the busy bit is held by another task.
         */
        atomic_t busy ____cacheline_aligned_in_smp;
        /* Consumer and producer counters are put into separate pages to
         * allow each position to be mapped with different permissions.
         * This prevents a user-space application from modifying the
         * position and ruining in-kernel tracking. The permissions of the
         * pages depend on who is producing samples: user-space or the
         * kernel. Note that the pending counter is placed in the same
         * page as the producer, so that it shares the same cache line.
         *
         * Kernel-producer
         * ---------------
         * The producer position and data pages are mapped as r/o in
         * userspace. For this approach, bits in the header of samples are
         * used to signal to user-space, and to other producers, whether a
         * sample is currently being written.
         *
         * User-space producer
         * -------------------
         * Only the page containing the consumer position is mapped r/o in
         * user-space. User-space producers also use bits of the header to
         * communicate to the kernel, but the kernel must carefully check and
         * validate each sample to ensure that they're correctly formatted, and
         * fully contained within the ring buffer.
         */
        unsigned long consumer_pos __aligned(PAGE_SIZE);
        unsigned long producer_pos __aligned(PAGE_SIZE);
        unsigned long pending_pos;
        unsigned long overwrite_pos; /* position after the last overwritten record */
        char data[] __aligned(PAGE_SIZE);
};

struct bpf_ringbuf_map {
        struct bpf_map map;
        struct bpf_ringbuf *rb;
};

/* 8-byte ring buffer record header structure */
struct bpf_ringbuf_hdr {
        u32 len;
        u32 pg_off;
};

static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node)
{
        const gfp_t flags = GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL |
                            __GFP_NOWARN | __GFP_ZERO;
        int nr_meta_pages = RINGBUF_NR_META_PAGES;
        int nr_data_pages = data_sz >> PAGE_SHIFT;
        int nr_pages = nr_meta_pages + nr_data_pages;
        struct page **pages, *page;
        struct bpf_ringbuf *rb;
        size_t array_size;
        int i;

        /* Each data page is mapped twice to allow "virtual"
         * continuous read of samples wrapping around the end of ring
         * buffer area:
         * ------------------------------------------------------
         * | meta pages |  real data pages  |  same data pages  |
         * ------------------------------------------------------
         * |            | 1 2 3 4 5 6 7 8 9 | 1 2 3 4 5 6 7 8 9 |
         * ------------------------------------------------------
         * |            | TA             DA | TA             DA |
         * ------------------------------------------------------
         *                               ^^^^^^^
         *                                  |
         * Here, no need to worry about special handling of wrapped-around
         * data due to double-mapped data pages. This works both in kernel and
         * when mmap()'ed in user-space, simplifying both kernel and
         * user-space implementations significantly.
         */
        array_size = (nr_meta_pages + 2 * nr_data_pages) * sizeof(*pages);
        pages = bpf_map_area_alloc(array_size, numa_node);
        if (!pages)
                return NULL;

        for (i = 0; i < nr_pages; i++) {
                page = alloc_pages_node(numa_node, flags, 0);
                if (!page) {
                        nr_pages = i;
                        goto err_free_pages;
                }
                pages[i] = page;
                if (i >= nr_meta_pages)
                        pages[nr_data_pages + i] = page;
        }

        rb = vmap(pages, nr_meta_pages + 2 * nr_data_pages,
                  VM_MAP | VM_USERMAP, PAGE_KERNEL);
        if (rb) {
                kmemleak_not_leak(pages);
                rb->pages = pages;
                rb->nr_pages = nr_pages;
                return rb;
        }

err_free_pages:
        for (i = 0; i < nr_pages; i++)
                __free_page(pages[i]);
        bpf_map_area_free(pages);
        return NULL;
}

static void bpf_ringbuf_notify(struct irq_work *work)
{
        struct bpf_ringbuf *rb = container_of(work, struct bpf_ringbuf, work);

        wake_up_all(&rb->waitq);
}

/* Maximum size of ring buffer area is limited by 32-bit page offset within
 * record header, counted in pages. Reserve 8 bits for extensibility, and
 * take into account few extra pages for consumer/producer pages and
 * non-mmap()'able parts, the current maximum size would be:
 *
 *     (((1ULL << 24) - RINGBUF_POS_PAGES - RINGBUF_PGOFF) * PAGE_SIZE)
 *
 * This gives 64GB limit, which seems plenty for single ring buffer. Now
 * considering that the maximum value of data_sz is (4GB - 1), there
 * will be no overflow, so just note the size limit in the comments.
 */
static struct bpf_ringbuf *bpf_ringbuf_alloc(size_t data_sz, int numa_node, bool overwrite_mode)
{
        struct bpf_ringbuf *rb;

        rb = bpf_ringbuf_area_alloc(data_sz, numa_node);
        if (!rb)
                return NULL;

        raw_res_spin_lock_init(&rb->spinlock);
        atomic_set(&rb->busy, 0);
        init_waitqueue_head(&rb->waitq);
        init_irq_work(&rb->work, bpf_ringbuf_notify);

        rb->mask = data_sz - 1;
        rb->consumer_pos = 0;
        rb->producer_pos = 0;
        rb->pending_pos = 0;
        rb->overwrite_mode = overwrite_mode;

        return rb;
}

static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr)
{
        bool overwrite_mode = false;
        struct bpf_ringbuf_map *rb_map;

        if (attr->map_flags & ~RINGBUF_CREATE_FLAG_MASK)
                return ERR_PTR(-EINVAL);

        if (attr->map_flags & BPF_F_RB_OVERWRITE) {
                if (attr->map_type != BPF_MAP_TYPE_RINGBUF)
                        return ERR_PTR(-EINVAL);
                overwrite_mode = true;
        }

        if (attr->key_size || attr->value_size ||
            !is_power_of_2(attr->max_entries) ||
            !PAGE_ALIGNED(attr->max_entries))
                return ERR_PTR(-EINVAL);

        rb_map = bpf_map_area_alloc(sizeof(*rb_map), NUMA_NO_NODE);
        if (!rb_map)
                return ERR_PTR(-ENOMEM);

        bpf_map_init_from_attr(&rb_map->map, attr);

        rb_map->rb = bpf_ringbuf_alloc(attr->max_entries, rb_map->map.numa_node, overwrite_mode);
        if (!rb_map->rb) {
                bpf_map_area_free(rb_map);
                return ERR_PTR(-ENOMEM);
        }

        return &rb_map->map;
}

static void bpf_ringbuf_free(struct bpf_ringbuf *rb)
{
        irq_work_sync(&rb->work);

        /* copy pages pointer and nr_pages to local variable, as we are going
         * to unmap rb itself with vunmap() below
         */
        struct page **pages = rb->pages;
        int i, nr_pages = rb->nr_pages;

        vunmap(rb);
        for (i = 0; i < nr_pages; i++)
                __free_page(pages[i]);
        bpf_map_area_free(pages);
}

static void ringbuf_map_free(struct bpf_map *map)
{
        struct bpf_ringbuf_map *rb_map;

        rb_map = container_of(map, struct bpf_ringbuf_map, map);
        bpf_ringbuf_free(rb_map->rb);
        bpf_map_area_free(rb_map);
}

static void *ringbuf_map_lookup_elem(struct bpf_map *map, void *key)
{
        return ERR_PTR(-ENOTSUPP);
}

static long ringbuf_map_update_elem(struct bpf_map *map, void *key, void *value,
                                    u64 flags)
{
        return -ENOTSUPP;
}

static long ringbuf_map_delete_elem(struct bpf_map *map, void *key)
{
        return -ENOTSUPP;
}

static int ringbuf_map_get_next_key(struct bpf_map *map, void *key,
                                    void *next_key)
{
        return -ENOTSUPP;
}

static int ringbuf_map_mmap_kern(struct bpf_map *map, struct vm_area_struct *vma)
{
        struct bpf_ringbuf_map *rb_map;

        rb_map = container_of(map, struct bpf_ringbuf_map, map);

        if (vma->vm_flags & VM_WRITE) {
                /* allow writable mapping for the consumer_pos only */
                if (vma->vm_pgoff != 0 || vma->vm_end - vma->vm_start != PAGE_SIZE)
                        return -EPERM;
        }
        /* remap_vmalloc_range() checks size and offset constraints */
        return remap_vmalloc_range(vma, rb_map->rb,
                                   vma->vm_pgoff + RINGBUF_PGOFF);
}

static int ringbuf_map_mmap_user(struct bpf_map *map, struct vm_area_struct *vma)
{
        struct bpf_ringbuf_map *rb_map;

        rb_map = container_of(map, struct bpf_ringbuf_map, map);

        if (vma->vm_flags & VM_WRITE) {
                if (vma->vm_pgoff == 0)
                        /* Disallow writable mappings to the consumer pointer,
                         * and allow writable mappings to both the producer
                         * position, and the ring buffer data itself.
                         */
                        return -EPERM;
        }
        /* remap_vmalloc_range() checks size and offset constraints */
        return remap_vmalloc_range(vma, rb_map->rb, vma->vm_pgoff + RINGBUF_PGOFF);
}

/*
 * Return an estimate of the available data in the ring buffer.
 * Note: the returned value can exceed the actual ring buffer size because the
 * function is not synchronized with the producer. The producer acquires the
 * ring buffer's spinlock, but this function does not.
 */
static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb)
{
        unsigned long cons_pos, prod_pos, over_pos;

        cons_pos = smp_load_acquire(&rb->consumer_pos);

        if (unlikely(rb->overwrite_mode)) {
                over_pos = smp_load_acquire(&rb->overwrite_pos);
                prod_pos = smp_load_acquire(&rb->producer_pos);
                return prod_pos - max(cons_pos, over_pos);
        } else {
                prod_pos = smp_load_acquire(&rb->producer_pos);
                return prod_pos - cons_pos;
        }
}

static u32 ringbuf_total_data_sz(const struct bpf_ringbuf *rb)
{
        return rb->mask + 1;
}

static __poll_t ringbuf_map_poll_kern(struct bpf_map *map, struct file *filp,
                                      struct poll_table_struct *pts)
{
        struct bpf_ringbuf_map *rb_map;

        rb_map = container_of(map, struct bpf_ringbuf_map, map);
        poll_wait(filp, &rb_map->rb->waitq, pts);

        if (ringbuf_avail_data_sz(rb_map->rb))
                return EPOLLIN | EPOLLRDNORM;
        return 0;
}

static __poll_t ringbuf_map_poll_user(struct bpf_map *map, struct file *filp,
                                      struct poll_table_struct *pts)
{
        struct bpf_ringbuf_map *rb_map;

        rb_map = container_of(map, struct bpf_ringbuf_map, map);
        poll_wait(filp, &rb_map->rb->waitq, pts);

        if (ringbuf_avail_data_sz(rb_map->rb) < ringbuf_total_data_sz(rb_map->rb))
                return EPOLLOUT | EPOLLWRNORM;
        return 0;
}

static u64 ringbuf_map_mem_usage(const struct bpf_map *map)
{
        struct bpf_ringbuf *rb;
        int nr_data_pages;
        int nr_meta_pages;
        u64 usage = sizeof(struct bpf_ringbuf_map);

        rb = container_of(map, struct bpf_ringbuf_map, map)->rb;
        usage += (u64)rb->nr_pages << PAGE_SHIFT;
        nr_meta_pages = RINGBUF_NR_META_PAGES;
        nr_data_pages = map->max_entries >> PAGE_SHIFT;
        usage += (nr_meta_pages + 2 * nr_data_pages) * sizeof(struct page *);
        return usage;
}

BTF_ID_LIST_SINGLE(ringbuf_map_btf_ids, struct, bpf_ringbuf_map)
const struct bpf_map_ops ringbuf_map_ops = {
        .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = ringbuf_map_alloc,
        .map_free = ringbuf_map_free,
        .map_mmap = ringbuf_map_mmap_kern,
        .map_poll = ringbuf_map_poll_kern,
        .map_lookup_elem = ringbuf_map_lookup_elem,
        .map_update_elem = ringbuf_map_update_elem,
        .map_delete_elem = ringbuf_map_delete_elem,
        .map_get_next_key = ringbuf_map_get_next_key,
        .map_mem_usage = ringbuf_map_mem_usage,
        .map_btf_id = &ringbuf_map_btf_ids[0],
};

BTF_ID_LIST_SINGLE(user_ringbuf_map_btf_ids, struct, bpf_ringbuf_map)
const struct bpf_map_ops user_ringbuf_map_ops = {
        .map_meta_equal = bpf_map_meta_equal,
        .map_alloc = ringbuf_map_alloc,
        .map_free = ringbuf_map_free,
        .map_mmap = ringbuf_map_mmap_user,
        .map_poll = ringbuf_map_poll_user,
        .map_lookup_elem = ringbuf_map_lookup_elem,
        .map_update_elem = ringbuf_map_update_elem,
        .map_delete_elem = ringbuf_map_delete_elem,
        .map_get_next_key = ringbuf_map_get_next_key,
        .map_mem_usage = ringbuf_map_mem_usage,
        .map_btf_id = &user_ringbuf_map_btf_ids[0],
};

/* Given pointer to ring buffer record metadata and struct bpf_ringbuf itself,
 * calculate offset from record metadata to ring buffer in pages, rounded
 * down. This page offset is stored as part of record metadata and allows to
 * restore struct bpf_ringbuf * from record pointer. This page offset is
 * stored at offset 4 of record metadata header.
 */
static size_t bpf_ringbuf_rec_pg_off(struct bpf_ringbuf *rb,
                                     struct bpf_ringbuf_hdr *hdr)
{
        return ((void *)hdr - (void *)rb) >> PAGE_SHIFT;
}

/* Given pointer to ring buffer record header, restore pointer to struct
 * bpf_ringbuf itself by using page offset stored at offset 4
 */
static struct bpf_ringbuf *
bpf_ringbuf_restore_from_rec(struct bpf_ringbuf_hdr *hdr)
{
        unsigned long addr = (unsigned long)(void *)hdr;
        unsigned long off = (unsigned long)hdr->pg_off << PAGE_SHIFT;

        return (void*)((addr & PAGE_MASK) - off);
}

static bool bpf_ringbuf_has_space(const struct bpf_ringbuf *rb,
                                  unsigned long new_prod_pos,
                                  unsigned long cons_pos,
                                  unsigned long pend_pos)
{
        /*
         * No space if oldest not yet committed record until the newest
         * record span more than (ringbuf_size - 1).
         */
        if (new_prod_pos - pend_pos > rb->mask)
                return false;

        /* Ok, we have space in overwrite mode */
        if (unlikely(rb->overwrite_mode))
                return true;

        /*
         * No space if producer position advances more than (ringbuf_size - 1)
         * ahead of consumer position when not in overwrite mode.
         */
        if (new_prod_pos - cons_pos > rb->mask)
                return false;

        return true;
}

static u32 bpf_ringbuf_round_up_hdr_len(u32 hdr_len)
{
        hdr_len &= ~BPF_RINGBUF_DISCARD_BIT;
        return round_up(hdr_len + BPF_RINGBUF_HDR_SZ, 8);
}

static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size)
{
        unsigned long cons_pos, prod_pos, new_prod_pos, pend_pos, over_pos, flags;
        struct bpf_ringbuf_hdr *hdr;
        u32 len, pg_off, hdr_len;

        if (unlikely(size > RINGBUF_MAX_RECORD_SZ))
                return NULL;

        len = round_up(size + BPF_RINGBUF_HDR_SZ, 8);
        if (len > ringbuf_total_data_sz(rb))
                return NULL;

        cons_pos = smp_load_acquire(&rb->consumer_pos);

        if (raw_res_spin_lock_irqsave(&rb->spinlock, flags))
                return NULL;

        pend_pos = rb->pending_pos;
        prod_pos = rb->producer_pos;
        new_prod_pos = prod_pos + len;

        while (pend_pos < prod_pos) {
                hdr = (void *)rb->data + (pend_pos & rb->mask);
                hdr_len = READ_ONCE(hdr->len);
                if (hdr_len & BPF_RINGBUF_BUSY_BIT)
                        break;
                pend_pos += bpf_ringbuf_round_up_hdr_len(hdr_len);
        }
        rb->pending_pos = pend_pos;

        if (!bpf_ringbuf_has_space(rb, new_prod_pos, cons_pos, pend_pos)) {
                raw_res_spin_unlock_irqrestore(&rb->spinlock, flags);
                return NULL;
        }

        /*
         * In overwrite mode, advance overwrite_pos when the ring buffer is full.
         * The key points are to stay on record boundaries and consume enough records
         * to fit the new one.
         */
        if (unlikely(rb->overwrite_mode)) {
                over_pos = rb->overwrite_pos;
                while (new_prod_pos - over_pos > rb->mask) {
                        hdr = (void *)rb->data + (over_pos & rb->mask);
                        hdr_len = READ_ONCE(hdr->len);
                        /*
                         * The bpf_ringbuf_has_space() check above ensures we won’t
                         * step over a record currently being worked on by another
                         * producer.
                         */
                        over_pos += bpf_ringbuf_round_up_hdr_len(hdr_len);
                }
                /*
                 * smp_store_release(&rb->producer_pos, new_prod_pos) at
                 * the end of the function ensures that when consumer sees
                 * the updated rb->producer_pos, it always sees the updated
                 * rb->overwrite_pos, so when consumer reads overwrite_pos
                 * after smp_load_acquire(r->producer_pos), the overwrite_pos
                 * will always be valid.
                 */
                WRITE_ONCE(rb->overwrite_pos, over_pos);
        }

        hdr = (void *)rb->data + (prod_pos & rb->mask);
        pg_off = bpf_ringbuf_rec_pg_off(rb, hdr);
        hdr->len = size | BPF_RINGBUF_BUSY_BIT;
        hdr->pg_off = pg_off;

        /* pairs with consumer's smp_load_acquire() */
        smp_store_release(&rb->producer_pos, new_prod_pos);

        raw_res_spin_unlock_irqrestore(&rb->spinlock, flags);

        return (void *)hdr + BPF_RINGBUF_HDR_SZ;
}

BPF_CALL_3(bpf_ringbuf_reserve, struct bpf_map *, map, u64, size, u64, flags)
{
        struct bpf_ringbuf_map *rb_map;

        if (unlikely(flags))
                return 0;

        rb_map = container_of(map, struct bpf_ringbuf_map, map);
        return (unsigned long)__bpf_ringbuf_reserve(rb_map->rb, size);
}

const struct bpf_func_proto bpf_ringbuf_reserve_proto = {
        .func                = bpf_ringbuf_reserve,
        .ret_type        = RET_PTR_TO_RINGBUF_MEM_OR_NULL,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_CONST_ALLOC_SIZE_OR_ZERO,
        .arg3_type        = ARG_ANYTHING,
};

static void bpf_ringbuf_commit(void *sample, u64 flags, bool discard)
{
        unsigned long rec_pos, cons_pos;
        struct bpf_ringbuf_hdr *hdr;
        struct bpf_ringbuf *rb;
        u32 new_len;

        hdr = sample - BPF_RINGBUF_HDR_SZ;
        rb = bpf_ringbuf_restore_from_rec(hdr);
        new_len = hdr->len ^ BPF_RINGBUF_BUSY_BIT;
        if (discard)
                new_len |= BPF_RINGBUF_DISCARD_BIT;

        /* update record header with correct final size prefix */
        xchg(&hdr->len, new_len);

        /* if consumer caught up and is waiting for our record, notify about
         * new data availability
         */
        rec_pos = (void *)hdr - (void *)rb->data;
        cons_pos = smp_load_acquire(&rb->consumer_pos) & rb->mask;

        if (flags & BPF_RB_FORCE_WAKEUP)
                irq_work_queue(&rb->work);
        else if (cons_pos == rec_pos && !(flags & BPF_RB_NO_WAKEUP))
                irq_work_queue(&rb->work);
}

BPF_CALL_2(bpf_ringbuf_submit, void *, sample, u64, flags)
{
        bpf_ringbuf_commit(sample, flags, false /* discard */);
        return 0;
}

const struct bpf_func_proto bpf_ringbuf_submit_proto = {
        .func                = bpf_ringbuf_submit,
        .ret_type        = RET_VOID,
        .arg1_type        = ARG_PTR_TO_RINGBUF_MEM | OBJ_RELEASE,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_2(bpf_ringbuf_discard, void *, sample, u64, flags)
{
        bpf_ringbuf_commit(sample, flags, true /* discard */);
        return 0;
}

const struct bpf_func_proto bpf_ringbuf_discard_proto = {
        .func                = bpf_ringbuf_discard,
        .ret_type        = RET_VOID,
        .arg1_type        = ARG_PTR_TO_RINGBUF_MEM | OBJ_RELEASE,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_ringbuf_output, struct bpf_map *, map, void *, data, u64, size,
           u64, flags)
{
        struct bpf_ringbuf_map *rb_map;
        void *rec;

        if (unlikely(flags & ~(BPF_RB_NO_WAKEUP | BPF_RB_FORCE_WAKEUP)))
                return -EINVAL;

        rb_map = container_of(map, struct bpf_ringbuf_map, map);
        rec = __bpf_ringbuf_reserve(rb_map->rb, size);
        if (!rec)
                return -EAGAIN;

        memcpy(rec, data, size);
        bpf_ringbuf_commit(rec, flags, false /* discard */);
        return 0;
}

const struct bpf_func_proto bpf_ringbuf_output_proto = {
        .func                = bpf_ringbuf_output,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_2(bpf_ringbuf_query, struct bpf_map *, map, u64, flags)
{
        struct bpf_ringbuf *rb;

        rb = container_of(map, struct bpf_ringbuf_map, map)->rb;

        switch (flags) {
        case BPF_RB_AVAIL_DATA:
                return ringbuf_avail_data_sz(rb);
        case BPF_RB_RING_SIZE:
                return ringbuf_total_data_sz(rb);
        case BPF_RB_CONS_POS:
                return smp_load_acquire(&rb->consumer_pos);
        case BPF_RB_PROD_POS:
                return smp_load_acquire(&rb->producer_pos);
        case BPF_RB_OVERWRITE_POS:
                return smp_load_acquire(&rb->overwrite_pos);
        default:
                return 0;
        }
}

const struct bpf_func_proto bpf_ringbuf_query_proto = {
        .func                = bpf_ringbuf_query,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_ringbuf_reserve_dynptr, struct bpf_map *, map, u32, size, u64, flags,
           struct bpf_dynptr_kern *, ptr)
{
        struct bpf_ringbuf_map *rb_map;
        void *sample;
        int err;

        if (unlikely(flags)) {
                bpf_dynptr_set_null(ptr);
                return -EINVAL;
        }

        err = bpf_dynptr_check_size(size);
        if (err) {
                bpf_dynptr_set_null(ptr);
                return err;
        }

        rb_map = container_of(map, struct bpf_ringbuf_map, map);

        sample = __bpf_ringbuf_reserve(rb_map->rb, size);
        if (!sample) {
                bpf_dynptr_set_null(ptr);
                return -EINVAL;
        }

        bpf_dynptr_init(ptr, sample, BPF_DYNPTR_TYPE_RINGBUF, 0, size);

        return 0;
}

const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto = {
        .func                = bpf_ringbuf_reserve_dynptr,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | MEM_UNINIT | MEM_WRITE,
};

BPF_CALL_2(bpf_ringbuf_submit_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags)
{
        if (!ptr->data)
                return 0;

        bpf_ringbuf_commit(ptr->data, flags, false /* discard */);

        bpf_dynptr_set_null(ptr);

        return 0;
}

const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto = {
        .func                = bpf_ringbuf_submit_dynptr,
        .ret_type        = RET_VOID,
        .arg1_type        = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_2(bpf_ringbuf_discard_dynptr, struct bpf_dynptr_kern *, ptr, u64, flags)
{
        if (!ptr->data)
                return 0;

        bpf_ringbuf_commit(ptr->data, flags, true /* discard */);

        bpf_dynptr_set_null(ptr);

        return 0;
}

const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto = {
        .func                = bpf_ringbuf_discard_dynptr,
        .ret_type        = RET_VOID,
        .arg1_type        = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE,
        .arg2_type        = ARG_ANYTHING,
};

static int __bpf_user_ringbuf_peek(struct bpf_ringbuf *rb, void **sample, u32 *size)
{
        int err;
        u32 hdr_len, sample_len, total_len, flags, *hdr;
        u64 cons_pos, prod_pos;

        /* Synchronizes with smp_store_release() in user-space producer. */
        prod_pos = smp_load_acquire(&rb->producer_pos);
        if (prod_pos % 8)
                return -EINVAL;

        /* Synchronizes with smp_store_release() in __bpf_user_ringbuf_sample_release() */
        cons_pos = smp_load_acquire(&rb->consumer_pos);
        if (cons_pos >= prod_pos)
                return -ENODATA;

        hdr = (u32 *)((uintptr_t)rb->data + (uintptr_t)(cons_pos & rb->mask));
        /* Synchronizes with smp_store_release() in user-space producer. */
        hdr_len = smp_load_acquire(hdr);
        flags = hdr_len & (BPF_RINGBUF_BUSY_BIT | BPF_RINGBUF_DISCARD_BIT);
        sample_len = hdr_len & ~flags;
        total_len = round_up(sample_len + BPF_RINGBUF_HDR_SZ, 8);

        /* The sample must fit within the region advertised by the producer position. */
        if (total_len > prod_pos - cons_pos)
                return -EINVAL;

        /* The sample must fit within the data region of the ring buffer. */
        if (total_len > ringbuf_total_data_sz(rb))
                return -E2BIG;

        /* The sample must fit into a struct bpf_dynptr. */
        err = bpf_dynptr_check_size(sample_len);
        if (err)
                return -E2BIG;

        if (flags & BPF_RINGBUF_DISCARD_BIT) {
                /* If the discard bit is set, the sample should be skipped.
                 *
                 * Update the consumer pos, and return -EAGAIN so the caller
                 * knows to skip this sample and try to read the next one.
                 */
                smp_store_release(&rb->consumer_pos, cons_pos + total_len);
                return -EAGAIN;
        }

        if (flags & BPF_RINGBUF_BUSY_BIT)
                return -ENODATA;

        *sample = (void *)((uintptr_t)rb->data +
                           (uintptr_t)((cons_pos + BPF_RINGBUF_HDR_SZ) & rb->mask));
        *size = sample_len;
        return 0;
}

static void __bpf_user_ringbuf_sample_release(struct bpf_ringbuf *rb, size_t size, u64 flags)
{
        u64 consumer_pos;
        u32 rounded_size = round_up(size + BPF_RINGBUF_HDR_SZ, 8);

        /* Using smp_load_acquire() is unnecessary here, as the busy-bit
         * prevents another task from writing to consumer_pos after it was read
         * by this task with smp_load_acquire() in __bpf_user_ringbuf_peek().
         */
        consumer_pos = rb->consumer_pos;
         /* Synchronizes with smp_load_acquire() in user-space producer. */
        smp_store_release(&rb->consumer_pos, consumer_pos + rounded_size);
}

BPF_CALL_4(bpf_user_ringbuf_drain, struct bpf_map *, map,
           void *, callback_fn, void *, callback_ctx, u64, flags)
{
        struct bpf_ringbuf *rb;
        long samples, discarded_samples = 0, ret = 0;
        bpf_callback_t callback = (bpf_callback_t)callback_fn;
        u64 wakeup_flags = BPF_RB_NO_WAKEUP | BPF_RB_FORCE_WAKEUP;
        int busy = 0;

        if (unlikely(flags & ~wakeup_flags))
                return -EINVAL;

        rb = container_of(map, struct bpf_ringbuf_map, map)->rb;

        /* If another consumer is already consuming a sample, wait for them to finish. */
        if (!atomic_try_cmpxchg(&rb->busy, &busy, 1))
                return -EBUSY;

        for (samples = 0; samples < BPF_MAX_USER_RINGBUF_SAMPLES && ret == 0; samples++) {
                int err;
                u32 size;
                void *sample;
                struct bpf_dynptr_kern dynptr;

                err = __bpf_user_ringbuf_peek(rb, &sample, &size);
                if (err) {
                        if (err == -ENODATA) {
                                break;
                        } else if (err == -EAGAIN) {
                                discarded_samples++;
                                continue;
                        } else {
                                ret = err;
                                goto schedule_work_return;
                        }
                }

                bpf_dynptr_init(&dynptr, sample, BPF_DYNPTR_TYPE_LOCAL, 0, size);
                ret = callback((uintptr_t)&dynptr, (uintptr_t)callback_ctx, 0, 0, 0);
                __bpf_user_ringbuf_sample_release(rb, size, flags);
        }
        ret = samples - discarded_samples;

schedule_work_return:
        /* Prevent the clearing of the busy-bit from being reordered before the
         * storing of any rb consumer or producer positions.
         */
        atomic_set_release(&rb->busy, 0);

        if (flags & BPF_RB_FORCE_WAKEUP)
                irq_work_queue(&rb->work);
        else if (!(flags & BPF_RB_NO_WAKEUP) && samples > 0)
                irq_work_queue(&rb->work);
        return ret;
}

const struct bpf_func_proto bpf_user_ringbuf_drain_proto = {
        .func                = bpf_user_ringbuf_drain,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_PTR_TO_FUNC,
        .arg3_type        = ARG_PTR_TO_STACK_OR_NULL,
        .arg4_type        = ARG_ANYTHING,
};




















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_KDEV_T_H
#define _LINUX_KDEV_T_H

#include <uapi/linux/kdev_t.h>

#define MINORBITS        20
#define MINORMASK        ((1U << MINORBITS) - 1)

#define MAJOR(dev)        ((unsigned int) ((dev) >> MINORBITS))
#define MINOR(dev)        ((unsigned int) ((dev) & MINORMASK))
#define MKDEV(ma,mi)        (((ma) << MINORBITS) | (mi))

#define print_dev_t(buffer, dev)                                        \
        sprintf((buffer), "%u:%u\n", MAJOR(dev), MINOR(dev))

#define format_dev_t(buffer, dev)                                        \
        ({                                                                \
                sprintf(buffer, "%u:%u", MAJOR(dev), MINOR(dev));        \
                buffer;                                                        \
        })

/* acceptable for old filesystems */
static __always_inline bool old_valid_dev(dev_t dev)
{
        return MAJOR(dev) < 256 && MINOR(dev) < 256;
}

static __always_inline u16 old_encode_dev(dev_t dev)
{
        return (MAJOR(dev) << 8) | MINOR(dev);
}

static __always_inline dev_t old_decode_dev(u16 val)
{
        return MKDEV((val >> 8) & 255, val & 255);
}

static __always_inline u32 new_encode_dev(dev_t dev)
{
        unsigned major = MAJOR(dev);
        unsigned minor = MINOR(dev);
        return (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12);
}

static __always_inline dev_t new_decode_dev(u32 dev)
{
        unsigned major = (dev & 0xfff00) >> 8;
        unsigned minor = (dev & 0xff) | ((dev >> 12) & 0xfff00);
        return MKDEV(major, minor);
}

static __always_inline u64 huge_encode_dev(dev_t dev)
{
        return new_encode_dev(dev);
}

static __always_inline dev_t huge_decode_dev(u64 dev)
{
        return new_decode_dev(dev);
}

static __always_inline int sysv_valid_dev(dev_t dev)
{
        return MAJOR(dev) < (1<<14) && MINOR(dev) < (1<<18);
}

static __always_inline u32 sysv_encode_dev(dev_t dev)
{
        return MINOR(dev) | (MAJOR(dev) << 18);
}

static __always_inline unsigned sysv_major(u32 dev)
{
        return (dev >> 18) & 0x3fff;
}

static __always_inline unsigned sysv_minor(u32 dev)
{
        return dev & 0x3ffff;
}

#endif





































































   14 







    1 

































  117 















  344 
























































































































































































  318 






  321 










  320 


  320 










    1 





  319 

  317 






  149 
  183 





  321 












































































































































































  105 

    4 






  103 





  105 

  105 






















   10 





   10 







    9 
   10 


   10 

   10 


    1 
    9 







  105 










  105 

  105 
  105 



  105 






































  101 

    6 
















  105 


























    6 
  101 


  105 






  105 






























   14 

    1 







   14 




   10 







   10 












  387 










  388 
  388 
  389 
  386 





    5 





    5 





    5 


    5 






    5 


    5 




















































































































































   36 








   36 
   36 
   32 
    5 


   36 




   36 











































  913 


















  913 









   27 
  915 
  911 

  771 
  148 


  913 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
// SPDX-License-Identifier: GPL-2.0-only
/*
 * fs/kernfs/file.c - kernfs file implementation
 *
 * Copyright (c) 2001-3 Patrick Mochel
 * Copyright (c) 2007 SUSE Linux Products GmbH
 * Copyright (c) 2007, 2013 Tejun Heo <tj@kernel.org>
 */

#include <linux/fs.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/poll.h>
#include <linux/pagemap.h>
#include <linux/sched/mm.h>
#include <linux/fsnotify.h>
#include <linux/uio.h>

#include "kernfs-internal.h"

struct kernfs_open_node {
        struct rcu_head                rcu_head;
        atomic_t                event;
        wait_queue_head_t        poll;
        struct list_head        files; /* goes through kernfs_open_file.list */
        unsigned int                nr_mmapped;
        unsigned int                nr_to_release;
};

/*
 * kernfs_notify() may be called from any context and bounces notifications
 * through a work item.  To minimize space overhead in kernfs_node, the
 * pending queue is implemented as a singly linked list of kernfs_nodes.
 * The list is terminated with the self pointer so that whether a
 * kernfs_node is on the list or not can be determined by testing the next
 * pointer for %NULL.
 */
#define KERNFS_NOTIFY_EOL                        ((void *)&kernfs_notify_list)

static DEFINE_SPINLOCK(kernfs_notify_lock);
static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL;

static inline struct mutex *kernfs_open_file_mutex_ptr(struct kernfs_node *kn)
{
        int idx = hash_ptr(kn, NR_KERNFS_LOCK_BITS);

        return &kernfs_locks->open_file_mutex[idx];
}

static inline struct mutex *kernfs_open_file_mutex_lock(struct kernfs_node *kn)
{
        struct mutex *lock;

        lock = kernfs_open_file_mutex_ptr(kn);

        mutex_lock(lock);

        return lock;
}

/**
 * of_on - Get the kernfs_open_node of the specified kernfs_open_file
 * @of: target kernfs_open_file
 *
 * Return: the kernfs_open_node of the kernfs_open_file
 */
static struct kernfs_open_node *of_on(struct kernfs_open_file *of)
{
        return rcu_dereference_protected(of->kn->attr.open,
                                         !list_empty(&of->list));
}

/* Get active reference to kernfs node for an open file */
static struct kernfs_open_file *kernfs_get_active_of(struct kernfs_open_file *of)
{
        /* Skip if file was already released */
        if (unlikely(of->released))
                return NULL;

        if (!kernfs_get_active(of->kn))
                return NULL;

        return of;
}

static void kernfs_put_active_of(struct kernfs_open_file *of)
{
        return kernfs_put_active(of->kn);
}

/**
 * kernfs_deref_open_node_locked - Get kernfs_open_node corresponding to @kn
 *
 * @kn: target kernfs_node.
 *
 * Fetch and return ->attr.open of @kn when caller holds the
 * kernfs_open_file_mutex_ptr(kn).
 *
 * Update of ->attr.open happens under kernfs_open_file_mutex_ptr(kn). So when
 * the caller guarantees that this mutex is being held, other updaters can't
 * change ->attr.open and this means that we can safely deref ->attr.open
 * outside RCU read-side critical section.
 *
 * The caller needs to make sure that kernfs_open_file_mutex is held.
 *
 * Return: @kn->attr.open when kernfs_open_file_mutex is held.
 */
static struct kernfs_open_node *
kernfs_deref_open_node_locked(struct kernfs_node *kn)
{
        return rcu_dereference_protected(kn->attr.open,
                                lockdep_is_held(kernfs_open_file_mutex_ptr(kn)));
}

static struct kernfs_open_file *kernfs_of(struct file *file)
{
        return ((struct seq_file *)file->private_data)->private;
}

/*
 * Determine the kernfs_ops for the given kernfs_node.  This function must
 * be called while holding an active reference.
 */
static const struct kernfs_ops *kernfs_ops(struct kernfs_node *kn)
{
        if (kn->flags & KERNFS_LOCKDEP)
                lockdep_assert_held(kn);
        return kn->attr.ops;
}

/*
 * As kernfs_seq_stop() is also called after kernfs_seq_start() or
 * kernfs_seq_next() failure, it needs to distinguish whether it's stopping
 * a seq_file iteration which is fully initialized with an active reference
 * or an aborted kernfs_seq_start() due to get_active failure.  The
 * position pointer is the only context for each seq_file iteration and
 * thus the stop condition should be encoded in it.  As the return value is
 * directly visible to userland, ERR_PTR(-ENODEV) is the only acceptable
 * choice to indicate get_active failure.
 *
 * Unfortunately, this is complicated due to the optional custom seq_file
 * operations which may return ERR_PTR(-ENODEV) too.  kernfs_seq_stop()
 * can't distinguish whether ERR_PTR(-ENODEV) is from get_active failure or
 * custom seq_file operations and thus can't decide whether put_active
 * should be performed or not only on ERR_PTR(-ENODEV).
 *
 * This is worked around by factoring out the custom seq_stop() and
 * put_active part into kernfs_seq_stop_active(), skipping it from
 * kernfs_seq_stop() if ERR_PTR(-ENODEV) while invoking it directly after
 * custom seq_file operations fail with ERR_PTR(-ENODEV) - this ensures
 * that kernfs_seq_stop_active() is skipped only after get_active failure.
 */
static void kernfs_seq_stop_active(struct seq_file *sf, void *v)
{
        struct kernfs_open_file *of = sf->private;
        const struct kernfs_ops *ops = kernfs_ops(of->kn);

        if (ops->seq_stop)
                ops->seq_stop(sf, v);
        kernfs_put_active_of(of);
}

static void *kernfs_seq_start(struct seq_file *sf, loff_t *ppos)
{
        struct kernfs_open_file *of = sf->private;
        const struct kernfs_ops *ops;

        /*
         * @of->mutex nests outside active ref and is primarily to ensure that
         * the ops aren't called concurrently for the same open file.
         */
        mutex_lock(&of->mutex);
        if (!kernfs_get_active_of(of))
                return ERR_PTR(-ENODEV);

        ops = kernfs_ops(of->kn);
        if (ops->seq_start) {
                void *next = ops->seq_start(sf, ppos);
                /* see the comment above kernfs_seq_stop_active() */
                if (next == ERR_PTR(-ENODEV))
                        kernfs_seq_stop_active(sf, next);
                return next;
        }
        return single_start(sf, ppos);
}

static void *kernfs_seq_next(struct seq_file *sf, void *v, loff_t *ppos)
{
        struct kernfs_open_file *of = sf->private;
        const struct kernfs_ops *ops = kernfs_ops(of->kn);

        if (ops->seq_next) {
                void *next = ops->seq_next(sf, v, ppos);
                /* see the comment above kernfs_seq_stop_active() */
                if (next == ERR_PTR(-ENODEV))
                        kernfs_seq_stop_active(sf, next);
                return next;
        } else {
                /*
                 * The same behavior and code as single_open(), always
                 * terminate after the initial read.
                 */
                ++*ppos;
                return NULL;
        }
}

static void kernfs_seq_stop(struct seq_file *sf, void *v)
{
        struct kernfs_open_file *of = sf->private;

        if (v != ERR_PTR(-ENODEV))
                kernfs_seq_stop_active(sf, v);
        mutex_unlock(&of->mutex);
}

static int kernfs_seq_show(struct seq_file *sf, void *v)
{
        struct kernfs_open_file *of = sf->private;

        of->event = atomic_read(&of_on(of)->event);

        return of->kn->attr.ops->seq_show(sf, v);
}

static const struct seq_operations kernfs_seq_ops = {
        .start = kernfs_seq_start,
        .next = kernfs_seq_next,
        .stop = kernfs_seq_stop,
        .show = kernfs_seq_show,
};

/*
 * As reading a bin file can have side-effects, the exact offset and bytes
 * specified in read(2) call should be passed to the read callback making
 * it difficult to use seq_file.  Implement simplistic custom buffering for
 * bin files.
 */
static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
{
        struct kernfs_open_file *of = kernfs_of(iocb->ki_filp);
        ssize_t len = min_t(size_t, iov_iter_count(iter), PAGE_SIZE);
        const struct kernfs_ops *ops;
        char *buf;

        buf = of->prealloc_buf;
        if (buf)
                mutex_lock(&of->prealloc_mutex);
        else
                buf = kmalloc(len, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;

        /*
         * @of->mutex nests outside active ref and is used both to ensure that
         * the ops aren't called concurrently for the same open file.
         */
        mutex_lock(&of->mutex);
        if (!kernfs_get_active_of(of)) {
                len = -ENODEV;
                mutex_unlock(&of->mutex);
                goto out_free;
        }

        of->event = atomic_read(&of_on(of)->event);

        ops = kernfs_ops(of->kn);
        if (ops->read)
                len = ops->read(of, buf, len, iocb->ki_pos);
        else
                len = -EINVAL;

        kernfs_put_active_of(of);
        mutex_unlock(&of->mutex);

        if (len < 0)
                goto out_free;

        if (copy_to_iter(buf, len, iter) != len) {
                len = -EFAULT;
                goto out_free;
        }

        iocb->ki_pos += len;

 out_free:
        if (buf == of->prealloc_buf)
                mutex_unlock(&of->prealloc_mutex);
        else
                kfree(buf);
        return len;
}

static ssize_t kernfs_fop_read_iter(struct kiocb *iocb, struct iov_iter *iter)
{
        if (kernfs_of(iocb->ki_filp)->kn->flags & KERNFS_HAS_SEQ_SHOW)
                return seq_read_iter(iocb, iter);
        return kernfs_file_read_iter(iocb, iter);
}

/*
 * Copy data in from userland and pass it to the matching kernfs write
 * operation.
 *
 * There is no easy way for us to know if userspace is only doing a partial
 * write, so we don't support them. We expect the entire buffer to come on
 * the first write.  Hint: if you're writing a value, first read the file,
 * modify only the value you're changing, then write entire buffer
 * back.
 */
static ssize_t kernfs_fop_write_iter(struct kiocb *iocb, struct iov_iter *iter)
{
        struct kernfs_open_file *of = kernfs_of(iocb->ki_filp);
        ssize_t len = iov_iter_count(iter);
        const struct kernfs_ops *ops;
        char *buf;

        if (of->atomic_write_len) {
                if (len > of->atomic_write_len)
                        return -E2BIG;
        } else {
                len = min_t(size_t, len, PAGE_SIZE);
        }

        buf = of->prealloc_buf;
        if (buf)
                mutex_lock(&of->prealloc_mutex);
        else
                buf = kmalloc(len + 1, GFP_KERNEL);
        if (!buf)
                return -ENOMEM;

        if (copy_from_iter(buf, len, iter) != len) {
                len = -EFAULT;
                goto out_free;
        }
        buf[len] = '\0';        /* guarantee string termination */

        /*
         * @of->mutex nests outside active ref and is used both to ensure that
         * the ops aren't called concurrently for the same open file.
         */
        mutex_lock(&of->mutex);
        if (!kernfs_get_active_of(of)) {
                mutex_unlock(&of->mutex);
                len = -ENODEV;
                goto out_free;
        }

        ops = kernfs_ops(of->kn);
        if (ops->write)
                len = ops->write(of, buf, len, iocb->ki_pos);
        else
                len = -EINVAL;

        kernfs_put_active_of(of);
        mutex_unlock(&of->mutex);

        if (len > 0)
                iocb->ki_pos += len;

out_free:
        if (buf == of->prealloc_buf)
                mutex_unlock(&of->prealloc_mutex);
        else
                kfree(buf);
        return len;
}

static void kernfs_vma_open(struct vm_area_struct *vma)
{
        struct file *file = vma->vm_file;
        struct kernfs_open_file *of = kernfs_of(file);

        if (!of->vm_ops)
                return;

        if (!kernfs_get_active_of(of))
                return;

        if (of->vm_ops->open)
                of->vm_ops->open(vma);

        kernfs_put_active_of(of);
}

static vm_fault_t kernfs_vma_fault(struct vm_fault *vmf)
{
        struct file *file = vmf->vma->vm_file;
        struct kernfs_open_file *of = kernfs_of(file);
        vm_fault_t ret;

        if (!of->vm_ops)
                return VM_FAULT_SIGBUS;

        if (!kernfs_get_active_of(of))
                return VM_FAULT_SIGBUS;

        ret = VM_FAULT_SIGBUS;
        if (of->vm_ops->fault)
                ret = of->vm_ops->fault(vmf);

        kernfs_put_active_of(of);
        return ret;
}

static vm_fault_t kernfs_vma_page_mkwrite(struct vm_fault *vmf)
{
        struct file *file = vmf->vma->vm_file;
        struct kernfs_open_file *of = kernfs_of(file);
        vm_fault_t ret;

        if (!of->vm_ops)
                return VM_FAULT_SIGBUS;

        if (!kernfs_get_active_of(of))
                return VM_FAULT_SIGBUS;

        ret = 0;
        if (of->vm_ops->page_mkwrite)
                ret = of->vm_ops->page_mkwrite(vmf);
        else
                file_update_time(file);

        kernfs_put_active_of(of);
        return ret;
}

static int kernfs_vma_access(struct vm_area_struct *vma, unsigned long addr,
                             void *buf, int len, int write)
{
        struct file *file = vma->vm_file;
        struct kernfs_open_file *of = kernfs_of(file);
        int ret;

        if (!of->vm_ops)
                return -EINVAL;

        if (!kernfs_get_active_of(of))
                return -EINVAL;

        ret = -EINVAL;
        if (of->vm_ops->access)
                ret = of->vm_ops->access(vma, addr, buf, len, write);

        kernfs_put_active_of(of);
        return ret;
}

static const struct vm_operations_struct kernfs_vm_ops = {
        .open                = kernfs_vma_open,
        .fault                = kernfs_vma_fault,
        .page_mkwrite        = kernfs_vma_page_mkwrite,
        .access                = kernfs_vma_access,
};

static int kernfs_fop_mmap(struct file *file, struct vm_area_struct *vma)
{
        struct kernfs_open_file *of = kernfs_of(file);
        const struct kernfs_ops *ops;
        int rc;

        /*
         * mmap path and of->mutex are prone to triggering spurious lockdep
         * warnings and we don't want to add spurious locking dependency
         * between the two.  Check whether mmap is actually implemented
         * without grabbing @of->mutex by testing HAS_MMAP flag.  See the
         * comment in kernfs_fop_open() for more details.
         */
        if (!(of->kn->flags & KERNFS_HAS_MMAP))
                return -ENODEV;

        mutex_lock(&of->mutex);

        rc = -ENODEV;
        if (!kernfs_get_active_of(of))
                goto out_unlock;

        ops = kernfs_ops(of->kn);
        rc = ops->mmap(of, vma);
        if (rc)
                goto out_put;

        /*
         * PowerPC's pci_mmap of legacy_mem uses shmem_zero_setup()
         * to satisfy versions of X which crash if the mmap fails: that
         * substitutes a new vm_file, and we don't then want bin_vm_ops.
         */
        if (vma->vm_file != file)
                goto out_put;

        rc = -EINVAL;
        if (of->mmapped && of->vm_ops != vma->vm_ops)
                goto out_put;

        /*
         * It is not possible to successfully wrap close.
         * So error if someone is trying to use close.
         */
        if (vma->vm_ops && vma->vm_ops->close)
                goto out_put;

        rc = 0;
        if (!of->mmapped) {
                of->mmapped = true;
                of_on(of)->nr_mmapped++;
                of->vm_ops = vma->vm_ops;
        }
        vma->vm_ops = &kernfs_vm_ops;
out_put:
        kernfs_put_active_of(of);
out_unlock:
        mutex_unlock(&of->mutex);

        return rc;
}

/**
 *        kernfs_get_open_node - get or create kernfs_open_node
 *        @kn: target kernfs_node
 *        @of: kernfs_open_file for this instance of open
 *
 *        If @kn->attr.open exists, increment its reference count; otherwise,
 *        create one.  @of is chained to the files list.
 *
 *        Locking:
 *        Kernel thread context (may sleep).
 *
 *        Return:
 *        %0 on success, -errno on failure.
 */
static int kernfs_get_open_node(struct kernfs_node *kn,
                                struct kernfs_open_file *of)
{
        struct kernfs_open_node *on;
        struct mutex *mutex;

        mutex = kernfs_open_file_mutex_lock(kn);
        on = kernfs_deref_open_node_locked(kn);

        if (!on) {
                /* not there, initialize a new one */
                on = kzalloc_obj(*on);
                if (!on) {
                        mutex_unlock(mutex);
                        return -ENOMEM;
                }
                atomic_set(&on->event, 1);
                init_waitqueue_head(&on->poll);
                INIT_LIST_HEAD(&on->files);
                rcu_assign_pointer(kn->attr.open, on);
        }

        list_add_tail(&of->list, &on->files);
        if (kn->flags & KERNFS_HAS_RELEASE)
                on->nr_to_release++;

        mutex_unlock(mutex);
        return 0;
}

/**
 *        kernfs_unlink_open_file - Unlink @of from @kn.
 *
 *        @kn: target kernfs_node
 *        @of: associated kernfs_open_file
 *        @open_failed: ->open() failed, cancel ->release()
 *
 *        Unlink @of from list of @kn's associated open files. If list of
 *        associated open files becomes empty, disassociate and free
 *        kernfs_open_node.
 *
 *        LOCKING:
 *        None.
 */
static void kernfs_unlink_open_file(struct kernfs_node *kn,
                                    struct kernfs_open_file *of,
                                    bool open_failed)
{
        struct kernfs_open_node *on;
        struct mutex *mutex;

        mutex = kernfs_open_file_mutex_lock(kn);

        on = kernfs_deref_open_node_locked(kn);
        if (!on) {
                mutex_unlock(mutex);
                return;
        }

        if (of) {
                if (kn->flags & KERNFS_HAS_RELEASE) {
                        WARN_ON_ONCE(of->released == open_failed);
                        if (open_failed)
                                on->nr_to_release--;
                }
                if (of->mmapped)
                        on->nr_mmapped--;
                list_del(&of->list);
        }

        if (list_empty(&on->files)) {
                rcu_assign_pointer(kn->attr.open, NULL);
                kfree_rcu(on, rcu_head);
        }

        mutex_unlock(mutex);
}

static int kernfs_fop_open(struct inode *inode, struct file *file)
{
        struct kernfs_node *kn = inode->i_private;
        struct kernfs_root *root = kernfs_root(kn);
        const struct kernfs_ops *ops;
        struct kernfs_open_file *of;
        bool has_read, has_write, has_mmap;
        int error = -EACCES;

        if (!kernfs_get_active(kn))
                return -ENODEV;

        ops = kernfs_ops(kn);

        has_read = ops->seq_show || ops->read || ops->mmap;
        has_write = ops->write || ops->mmap;
        has_mmap = ops->mmap;

        /* see the flag definition for details */
        if (root->flags & KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK) {
                if ((file->f_mode & FMODE_WRITE) &&
                    (!(inode->i_mode & S_IWUGO) || !has_write))
                        goto err_out;

                if ((file->f_mode & FMODE_READ) &&
                    (!(inode->i_mode & S_IRUGO) || !has_read))
                        goto err_out;
        }

        /* allocate a kernfs_open_file for the file */
        error = -ENOMEM;
        of = kzalloc_obj(struct kernfs_open_file);
        if (!of)
                goto err_out;

        /*
         * The following is done to give a different lockdep key to
         * @of->mutex for files which implement mmap.  This is a rather
         * crude way to avoid false positive lockdep warning around
         * mm->mmap_lock - mmap nests @of->mutex under mm->mmap_lock and
         * reading /sys/block/sda/trace/act_mask grabs sr_mutex, under
         * which mm->mmap_lock nests, while holding @of->mutex.  As each
         * open file has a separate mutex, it's okay as long as those don't
         * happen on the same file.  At this point, we can't easily give
         * each file a separate locking class.  Let's differentiate on
         * whether the file has mmap or not for now.
         *
         * For similar reasons, writable and readonly files are given different
         * lockdep key, because the writable file /sys/power/resume may call vfs
         * lookup helpers for arbitrary paths and readonly files can be read by
         * overlayfs from vfs helpers when sysfs is a lower layer of overalyfs.
         *
         * All three cases look the same.  They're supposed to
         * look that way and give @of->mutex different static lockdep keys.
         */
        if (has_mmap)
                mutex_init(&of->mutex);
        else if (file->f_mode & FMODE_WRITE)
                mutex_init(&of->mutex);
        else
                mutex_init(&of->mutex);

        of->kn = kn;
        of->file = file;

        /*
         * Write path needs to atomic_write_len outside active reference.
         * Cache it in open_file.  See kernfs_fop_write_iter() for details.
         */
        of->atomic_write_len = ops->atomic_write_len;

        error = -EINVAL;
        /*
         * ->seq_show is incompatible with ->prealloc,
         * as seq_read does its own allocation.
         * ->read must be used instead.
         */
        if (ops->prealloc && ops->seq_show)
                goto err_free;
        if (ops->prealloc) {
                int len = of->atomic_write_len ?: PAGE_SIZE;
                of->prealloc_buf = kmalloc(len + 1, GFP_KERNEL);
                error = -ENOMEM;
                if (!of->prealloc_buf)
                        goto err_free;
                mutex_init(&of->prealloc_mutex);
        }

        /*
         * Always instantiate seq_file even if read access doesn't use
         * seq_file or is not requested.  This unifies private data access
         * and readable regular files are the vast majority anyway.
         */
        if (ops->seq_show)
                error = seq_open(file, &kernfs_seq_ops);
        else
                error = seq_open(file, NULL);
        if (error)
                goto err_free;

        of->seq_file = file->private_data;
        of->seq_file->private = of;

        /* seq_file clears PWRITE unconditionally, restore it if WRITE */
        if (file->f_mode & FMODE_WRITE)
                file->f_mode |= FMODE_PWRITE;

        /* make sure we have open node struct */
        error = kernfs_get_open_node(kn, of);
        if (error)
                goto err_seq_release;

        if (ops->open) {
                /* nobody has access to @of yet, skip @of->mutex */
                error = ops->open(of);
                if (error)
                        goto err_put_node;
        }

        /* open succeeded, put active references */
        kernfs_put_active(kn);
        return 0;

err_put_node:
        kernfs_unlink_open_file(kn, of, true);
err_seq_release:
        seq_release(inode, file);
err_free:
        kfree(of->prealloc_buf);
        kfree(of);
err_out:
        kernfs_put_active(kn);
        return error;
}

/* used from release/drain to ensure that ->release() is called exactly once */
static void kernfs_release_file(struct kernfs_node *kn,
                                struct kernfs_open_file *of)
{
        /*
         * @of is guaranteed to have no other file operations in flight and
         * we just want to synchronize release and drain paths.
         * @kernfs_open_file_mutex_ptr(kn) is enough. @of->mutex can't be used
         * here because drain path may be called from places which can
         * cause circular dependency.
         */
        lockdep_assert_held(kernfs_open_file_mutex_ptr(kn));

        if (!of->released) {
                /*
                 * A file is never detached without being released and we
                 * need to be able to release files which are deactivated
                 * and being drained.  Don't use kernfs_ops().
                 */
                kn->attr.ops->release(of);
                of->released = true;
                of_on(of)->nr_to_release--;
        }
}

static int kernfs_fop_release(struct inode *inode, struct file *filp)
{
        struct kernfs_node *kn = inode->i_private;
        struct kernfs_open_file *of = kernfs_of(filp);

        if (kn->flags & KERNFS_HAS_RELEASE) {
                struct mutex *mutex;

                mutex = kernfs_open_file_mutex_lock(kn);
                kernfs_release_file(kn, of);
                mutex_unlock(mutex);
        }

        kernfs_unlink_open_file(kn, of, false);
        seq_release(inode, filp);
        kfree(of->prealloc_buf);
        kfree(of);

        return 0;
}

bool kernfs_should_drain_open_files(struct kernfs_node *kn)
{
        struct kernfs_open_node *on;
        bool ret;

        /*
         * @kn being deactivated guarantees that @kn->attr.open can't change
         * beneath us making the lockless test below safe.
         * Callers post kernfs_unbreak_active_protection may be counted in
         * kn->active by now, do not WARN_ON because of them.
         */

        rcu_read_lock();
        on = rcu_dereference(kn->attr.open);
        ret = on && (on->nr_mmapped || on->nr_to_release);
        rcu_read_unlock();

        return ret;
}

void kernfs_drain_open_files(struct kernfs_node *kn)
{
        struct kernfs_open_node *on;
        struct kernfs_open_file *of;
        struct mutex *mutex;

        mutex = kernfs_open_file_mutex_lock(kn);
        on = kernfs_deref_open_node_locked(kn);
        if (!on) {
                mutex_unlock(mutex);
                return;
        }

        list_for_each_entry(of, &on->files, list) {
                struct inode *inode = file_inode(of->file);

                if (of->mmapped) {
                        unmap_mapping_range(inode->i_mapping, 0, 0, 1);
                        of->mmapped = false;
                        on->nr_mmapped--;
                }

                if (kn->flags & KERNFS_HAS_RELEASE)
                        kernfs_release_file(kn, of);
        }

        WARN_ON_ONCE(on->nr_mmapped || on->nr_to_release);
        mutex_unlock(mutex);
}

/*
 * Kernfs attribute files are pollable.  The idea is that you read
 * the content and then you use 'poll' or 'select' to wait for
 * the content to change.  When the content changes (assuming the
 * manager for the kobject supports notification), poll will
 * return EPOLLERR|EPOLLPRI, and select will return the fd whether
 * it is waiting for read, write, or exceptions.
 * Once poll/select indicates that the value has changed, you
 * need to close and re-open the file, or seek to 0 and read again.
 * Reminder: this only works for attributes which actively support
 * it, and it is not possible to test an attribute from userspace
 * to see if it supports poll (Neither 'poll' nor 'select' return
 * an appropriate error code).  When in doubt, set a suitable timeout value.
 */
__poll_t kernfs_generic_poll(struct kernfs_open_file *of, poll_table *wait)
{
        struct kernfs_open_node *on = of_on(of);

        poll_wait(of->file, &on->poll, wait);

        if (of->event != atomic_read(&on->event))
                return DEFAULT_POLLMASK|EPOLLERR|EPOLLPRI;

        return DEFAULT_POLLMASK;
}

static __poll_t kernfs_fop_poll(struct file *filp, poll_table *wait)
{
        struct kernfs_open_file *of = kernfs_of(filp);
        struct kernfs_node *kn = kernfs_dentry_node(filp->f_path.dentry);
        __poll_t ret;

        if (!kernfs_get_active_of(of))
                return DEFAULT_POLLMASK|EPOLLERR|EPOLLPRI;

        if (kn->attr.ops->poll)
                ret = kn->attr.ops->poll(of, wait);
        else
                ret = kernfs_generic_poll(of, wait);

        kernfs_put_active_of(of);
        return ret;
}

static loff_t kernfs_fop_llseek(struct file *file, loff_t offset, int whence)
{
        struct kernfs_open_file *of = kernfs_of(file);
        const struct kernfs_ops *ops;
        loff_t ret;

        /*
         * @of->mutex nests outside active ref and is primarily to ensure that
         * the ops aren't called concurrently for the same open file.
         */
        mutex_lock(&of->mutex);
        if (!kernfs_get_active_of(of)) {
                mutex_unlock(&of->mutex);
                return -ENODEV;
        }

        ops = kernfs_ops(of->kn);
        if (ops->llseek)
                ret = ops->llseek(of, offset, whence);
        else
                ret = generic_file_llseek(file, offset, whence);

        kernfs_put_active_of(of);
        mutex_unlock(&of->mutex);
        return ret;
}

static void kernfs_notify_workfn(struct work_struct *work)
{
        struct kernfs_node *kn;
        struct kernfs_super_info *info;
        struct kernfs_root *root;
repeat:
        /* pop one off the notify_list */
        spin_lock_irq(&kernfs_notify_lock);
        kn = kernfs_notify_list;
        if (kn == KERNFS_NOTIFY_EOL) {
                spin_unlock_irq(&kernfs_notify_lock);
                return;
        }
        kernfs_notify_list = kn->attr.notify_next;
        kn->attr.notify_next = NULL;
        spin_unlock_irq(&kernfs_notify_lock);

        root = kernfs_root(kn);
        /* kick fsnotify */

        down_read(&root->kernfs_supers_rwsem);
        down_read(&root->kernfs_rwsem);
        list_for_each_entry(info, &kernfs_root(kn)->supers, node) {
                struct kernfs_node *parent;
                struct inode *p_inode = NULL;
                const char *kn_name;
                struct inode *inode;
                struct qstr name;

                /*
                 * We want fsnotify_modify() on @kn but as the
                 * modifications aren't originating from userland don't
                 * have the matching @file available.  Look up the inodes
                 * and generate the events manually.
                 */
                inode = ilookup(info->sb, kernfs_ino(kn));
                if (!inode)
                        continue;

                kn_name = kernfs_rcu_name(kn);
                name = QSTR(kn_name);
                parent = kernfs_get_parent(kn);
                if (parent) {
                        p_inode = ilookup(info->sb, kernfs_ino(parent));
                        if (p_inode) {
                                fsnotify(FS_MODIFY | FS_EVENT_ON_CHILD,
                                         inode, FSNOTIFY_EVENT_INODE,
                                         p_inode, &name, inode, 0);
                                iput(p_inode);
                        }

                        kernfs_put(parent);
                }

                if (!p_inode)
                        fsnotify_inode(inode, FS_MODIFY);

                iput(inode);
        }

        up_read(&root->kernfs_rwsem);
        up_read(&root->kernfs_supers_rwsem);
        kernfs_put(kn);
        goto repeat;
}

/**
 * kernfs_notify - notify a kernfs file
 * @kn: file to notify
 *
 * Notify @kn such that poll(2) on @kn wakes up.  Maybe be called from any
 * context.
 */
void kernfs_notify(struct kernfs_node *kn)
{
        static DECLARE_WORK(kernfs_notify_work, kernfs_notify_workfn);
        unsigned long flags;
        struct kernfs_open_node *on;

        if (WARN_ON(kernfs_type(kn) != KERNFS_FILE))
                return;

        /* kick poll immediately */
        rcu_read_lock();
        on = rcu_dereference(kn->attr.open);
        if (on) {
                atomic_inc(&on->event);
                wake_up_interruptible(&on->poll);
        }
        rcu_read_unlock();

        /* schedule work to kick fsnotify */
        spin_lock_irqsave(&kernfs_notify_lock, flags);
        if (!kn->attr.notify_next) {
                kernfs_get(kn);
                kn->attr.notify_next = kernfs_notify_list;
                kernfs_notify_list = kn;
                schedule_work(&kernfs_notify_work);
        }
        spin_unlock_irqrestore(&kernfs_notify_lock, flags);
}
EXPORT_SYMBOL_GPL(kernfs_notify);

const struct file_operations kernfs_file_fops = {
        .read_iter        = kernfs_fop_read_iter,
        .write_iter        = kernfs_fop_write_iter,
        .llseek                = kernfs_fop_llseek,
        .mmap                = kernfs_fop_mmap,
        .open                = kernfs_fop_open,
        .release        = kernfs_fop_release,
        .poll                = kernfs_fop_poll,
        .fsync                = noop_fsync,
        .splice_read        = copy_splice_read,
        .splice_write        = iter_file_splice_write,
};

/**
 * __kernfs_create_file - kernfs internal function to create a file
 * @parent: directory to create the file in
 * @name: name of the file
 * @mode: mode of the file
 * @uid: uid of the file
 * @gid: gid of the file
 * @size: size of the file
 * @ops: kernfs operations for the file
 * @priv: private data for the file
 * @ns: optional namespace tag of the file
 * @key: lockdep key for the file's active_ref, %NULL to disable lockdep
 *
 * Return: the created node on success, ERR_PTR() value on error.
 */
struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent,
                                         const char *name,
                                         umode_t mode, kuid_t uid, kgid_t gid,
                                         loff_t size,
                                         const struct kernfs_ops *ops,
                                         void *priv, const struct ns_common *ns,
                                         struct lock_class_key *key)
{
        struct kernfs_node *kn;
        unsigned flags;
        int rc;

        flags = KERNFS_FILE;

        kn = kernfs_new_node(parent, name, (mode & S_IALLUGO) | S_IFREG,
                             uid, gid, flags);
        if (!kn)
                return ERR_PTR(-ENOMEM);

        kn->attr.ops = ops;
        kn->attr.size = size;
        kn->ns = ns;
        kn->priv = priv;

#ifdef CONFIG_DEBUG_LOCK_ALLOC
        if (key) {
                lockdep_init_map(&kn->dep_map, "kn->active", key, 0);
                kn->flags |= KERNFS_LOCKDEP;
        }
#endif

        /*
         * kn->attr.ops is accessible only while holding active ref.  We
         * need to know whether some ops are implemented outside active
         * ref.  Cache their existence in flags.
         */
        if (ops->seq_show)
                kn->flags |= KERNFS_HAS_SEQ_SHOW;
        if (ops->mmap)
                kn->flags |= KERNFS_HAS_MMAP;
        if (ops->release)
                kn->flags |= KERNFS_HAS_RELEASE;

        rc = kernfs_add_one(kn);
        if (rc) {
                kernfs_put(kn);
                return ERR_PTR(rc);
        }
        return kn;
}






































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM mmap

#if !defined(_TRACE_MMAP_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_MMAP_H

#include <linux/tracepoint.h>

TRACE_EVENT(vm_unmapped_area,

        TP_PROTO(unsigned long addr, struct vm_unmapped_area_info *info),

        TP_ARGS(addr, info),

        TP_STRUCT__entry(
                __field(unsigned long,        addr)
                __field(unsigned long,        total_vm)
                __field(unsigned long,        flags)
                __field(unsigned long,        length)
                __field(unsigned long,        low_limit)
                __field(unsigned long,        high_limit)
                __field(unsigned long,        align_mask)
                __field(unsigned long,        align_offset)
        ),

        TP_fast_assign(
                __entry->addr = addr;
                __entry->total_vm = current->mm->total_vm;
                __entry->flags = info->flags;
                __entry->length = info->length;
                __entry->low_limit = info->low_limit;
                __entry->high_limit = info->high_limit;
                __entry->align_mask = info->align_mask;
                __entry->align_offset = info->align_offset;
        ),

        TP_printk("addr=0x%lx err=%ld total_vm=0x%lx flags=0x%lx len=0x%lx lo=0x%lx hi=0x%lx mask=0x%lx ofs=0x%lx",
                IS_ERR_VALUE(__entry->addr) ? 0 : __entry->addr,
                IS_ERR_VALUE(__entry->addr) ? __entry->addr : 0,
                __entry->total_vm, __entry->flags, __entry->length,
                __entry->low_limit, __entry->high_limit, __entry->align_mask,
                __entry->align_offset)
);

TRACE_EVENT(exit_mmap,
        TP_PROTO(struct mm_struct *mm),

        TP_ARGS(mm),

        TP_STRUCT__entry(
                        __field(struct mm_struct *, mm)
                        __field(struct maple_tree *, mt)
        ),

        TP_fast_assign(
                       __entry->mm                = mm;
                       __entry->mt                = &mm->mm_mt;
        ),

        TP_printk("mt_mod %p, DESTROY",
                  __entry->mt
        )
);

#endif

/* This part must be outside protection */
#include <trace/define_trace.h>











































































































   10 











   10 






   10 
   10 





























































































































































































































   10 




























































   10 


   10 












































   10 















   10 












   10 








   10 























   10 












   10 




   10 







































































































































































































































































































   10 





















































   10 




















   10 











   10 






   10 

   10 

























   10 


   10 















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   10 











   10 





































   10 





















   10 


























































































































































































   10 





   10 

   10 










































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com
 * Written by Alex Tomas <alex@clusterfs.com>
 *
 * Architecture independence:
 *   Copyright (c) 2005, Bull S.A.
 *   Written by Pierre Peiffer <pierre.peiffer@bull.net>
 */

/*
 * Extents support for EXT4
 *
 * TODO:
 *   - ext4*_error() should be used in some situations
 *   - analyze all BUG()/BUG_ON(), use -EIO where appropriate
 *   - smart tree reduction
 */

#include <linux/fs.h>
#include <linux/time.h>
#include <linux/jbd2.h>
#include <linux/highuid.h>
#include <linux/pagemap.h>
#include <linux/quotaops.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/fiemap.h>
#include <linux/iomap.h>
#include <linux/sched/mm.h>
#include "ext4_jbd2.h"
#include "ext4_extents.h"
#include "xattr.h"
#include <kunit/static_stub.h>

#include <trace/events/ext4.h>

/*
 * used by extent splitting.
 */
#define EXT4_EXT_MAY_ZEROOUT        0x1  /* safe to zeroout if split fails \
                                        due to ENOSPC */
static struct ext4_ext_path *ext4_split_convert_extents(
        handle_t *handle, struct inode *inode, struct ext4_map_blocks *map,
        struct ext4_ext_path *path, int flags, unsigned int *allocated);

static __le32 ext4_extent_block_csum(struct inode *inode,
                                     struct ext4_extent_header *eh)
{
        struct ext4_inode_info *ei = EXT4_I(inode);
        __u32 csum;

        csum = ext4_chksum(ei->i_csum_seed, (__u8 *)eh,
                           EXT4_EXTENT_TAIL_OFFSET(eh));
        return cpu_to_le32(csum);
}

static int ext4_extent_block_csum_verify(struct inode *inode,
                                         struct ext4_extent_header *eh)
{
        struct ext4_extent_tail *et;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return 1;

        et = find_ext4_extent_tail(eh);
        if (et->et_checksum != ext4_extent_block_csum(inode, eh))
                return 0;
        return 1;
}

static void ext4_extent_block_csum_set(struct inode *inode,
                                       struct ext4_extent_header *eh)
{
        struct ext4_extent_tail *et;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return;

        et = find_ext4_extent_tail(eh);
        et->et_checksum = ext4_extent_block_csum(inode, eh);
}

static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle,
                                                  struct inode *inode,
                                                  struct ext4_ext_path *path,
                                                  ext4_lblk_t split, int flags);

static int ext4_ext_trunc_restart_fn(struct inode *inode, int *dropped)
{
        /*
         * Drop i_data_sem to avoid deadlock with ext4_map_blocks.  At this
         * moment, get_block can be called only for blocks inside i_size since
         * page cache has been already dropped and writes are blocked by
         * i_rwsem. So we can safely drop the i_data_sem here.
         */
        BUG_ON(EXT4_JOURNAL(inode) == NULL);
        ext4_discard_preallocations(inode);
        up_write(&EXT4_I(inode)->i_data_sem);
        *dropped = 1;
        return 0;
}

static inline void ext4_ext_path_brelse(struct ext4_ext_path *path)
{
        brelse(path->p_bh);
        path->p_bh = NULL;
}

static void ext4_ext_drop_refs(struct ext4_ext_path *path)
{
        int depth, i;

        if (IS_ERR_OR_NULL(path))
                return;
        depth = path->p_depth;
        for (i = 0; i <= depth; i++, path++)
                ext4_ext_path_brelse(path);
}

void ext4_free_ext_path(struct ext4_ext_path *path)
{
        if (IS_ERR_OR_NULL(path))
                return;
        ext4_ext_drop_refs(path);
        kfree(path);
}

/*
 * Make sure 'handle' has at least 'check_cred' credits. If not, restart
 * transaction with 'restart_cred' credits. The function drops i_data_sem
 * when restarting transaction and gets it after transaction is restarted.
 *
 * The function returns 0 on success, 1 if transaction had to be restarted,
 * and < 0 in case of fatal error.
 */
int ext4_datasem_ensure_credits(handle_t *handle, struct inode *inode,
                                int check_cred, int restart_cred,
                                int revoke_cred)
{
        int ret;
        int dropped = 0;

        ret = ext4_journal_ensure_credits_fn(handle, check_cred, restart_cred,
                revoke_cred, ext4_ext_trunc_restart_fn(inode, &dropped));
        if (dropped)
                down_write(&EXT4_I(inode)->i_data_sem);
        return ret;
}

/*
 * could return:
 *  - EROFS
 *  - ENOMEM
 */
static int ext4_ext_get_access(handle_t *handle, struct inode *inode,
                                struct ext4_ext_path *path)
{
        int err = 0;

        if (path->p_bh) {
                /* path points to block */
                BUFFER_TRACE(path->p_bh, "get_write_access");
                err = ext4_journal_get_write_access(handle, inode->i_sb,
                                                    path->p_bh, EXT4_JTR_NONE);
                /*
                 * The extent buffer's verified bit will be set again in
                 * __ext4_ext_dirty(). We could leave an inconsistent
                 * buffer if the extents updating procudure break off du
                 * to some error happens, force to check it again.
                 */
                if (!err)
                        clear_buffer_verified(path->p_bh);
        }
        /* path points to leaf/index in inode body */
        /* we use in-core data, no need to protect them */
        return err;
}

/*
 * could return:
 *  - EROFS
 *  - ENOMEM
 *  - EIO
 */
int __ext4_ext_dirty(const char *where, unsigned int line,
                     handle_t *handle, struct inode *inode,
                     struct ext4_ext_path *path)
{
        int err;

        KUNIT_STATIC_STUB_REDIRECT(__ext4_ext_dirty, where, line, handle, inode,
                                   path);

        WARN_ON(!rwsem_is_locked(&EXT4_I(inode)->i_data_sem));
        if (path->p_bh) {
                ext4_extent_block_csum_set(inode, ext_block_hdr(path->p_bh));
                /* path points to block */
                err = __ext4_handle_dirty_metadata(where, line, handle,
                                                   inode, path->p_bh);
                /* Extents updating done, re-set verified flag */
                if (!err)
                        set_buffer_verified(path->p_bh);
        } else {
                /* path points to leaf/index in inode body */
                err = ext4_mark_inode_dirty(handle, inode);
        }
        return err;
}

#define ext4_ext_dirty(handle, inode, path) \
                __ext4_ext_dirty(__func__, __LINE__, (handle), (inode), (path))

static ext4_fsblk_t ext4_ext_find_goal(struct inode *inode,
                              struct ext4_ext_path *path,
                              ext4_lblk_t block)
{
        if (path) {
                int depth = path->p_depth;
                struct ext4_extent *ex;

                /*
                 * Try to predict block placement assuming that we are
                 * filling in a file which will eventually be
                 * non-sparse --- i.e., in the case of libbfd writing
                 * an ELF object sections out-of-order but in a way
                 * the eventually results in a contiguous object or
                 * executable file, or some database extending a table
                 * space file.  However, this is actually somewhat
                 * non-ideal if we are writing a sparse file such as
                 * qemu or KVM writing a raw image file that is going
                 * to stay fairly sparse, since it will end up
                 * fragmenting the file system's free space.  Maybe we
                 * should have some hueristics or some way to allow
                 * userspace to pass a hint to file system,
                 * especially if the latter case turns out to be
                 * common.
                 */
                ex = path[depth].p_ext;
                if (ex) {
                        ext4_fsblk_t ext_pblk = ext4_ext_pblock(ex);
                        ext4_lblk_t ext_block = le32_to_cpu(ex->ee_block);

                        if (block > ext_block)
                                return ext_pblk + (block - ext_block);
                        else
                                return ext_pblk - (ext_block - block);
                }

                /* it looks like index is empty;
                 * try to find starting block from index itself */
                if (path[depth].p_bh)
                        return path[depth].p_bh->b_blocknr;
        }

        /* OK. use inode's group */
        return ext4_inode_to_goal_block(inode);
}

/*
 * Allocation for a meta data block
 */
static ext4_fsblk_t
ext4_ext_new_meta_block(handle_t *handle, struct inode *inode,
                        struct ext4_ext_path *path,
                        struct ext4_extent *ex, int *err, unsigned int flags)
{
        ext4_fsblk_t goal, newblock;

        goal = ext4_ext_find_goal(inode, path, le32_to_cpu(ex->ee_block));
        newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
                                        NULL, err);
        return newblock;
}

static inline int ext4_ext_space_block(struct inode *inode, int check)
{
        int size;

        size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
                        / sizeof(struct ext4_extent);
#ifdef AGGRESSIVE_TEST
        if (!check && size > 6)
                size = 6;
#endif
        return size;
}

static inline int ext4_ext_space_block_idx(struct inode *inode, int check)
{
        int size;

        size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
                        / sizeof(struct ext4_extent_idx);
#ifdef AGGRESSIVE_TEST
        if (!check && size > 5)
                size = 5;
#endif
        return size;
}

static inline int ext4_ext_space_root(struct inode *inode, int check)
{
        int size;

        size = sizeof(EXT4_I(inode)->i_data);
        size -= sizeof(struct ext4_extent_header);
        size /= sizeof(struct ext4_extent);
#ifdef AGGRESSIVE_TEST
        if (!check && size > 3)
                size = 3;
#endif
        return size;
}

static inline int ext4_ext_space_root_idx(struct inode *inode, int check)
{
        int size;

        size = sizeof(EXT4_I(inode)->i_data);
        size -= sizeof(struct ext4_extent_header);
        size /= sizeof(struct ext4_extent_idx);
#ifdef AGGRESSIVE_TEST
        if (!check && size > 4)
                size = 4;
#endif
        return size;
}

static inline struct ext4_ext_path *
ext4_force_split_extent_at(handle_t *handle, struct inode *inode,
                           struct ext4_ext_path *path, ext4_lblk_t lblk,
                           int nofail)
{
        int flags = EXT4_EX_NOCACHE | EXT4_GET_BLOCKS_SPLIT_NOMERGE;

        if (nofail)
                flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL | EXT4_EX_NOFAIL;

        return ext4_split_extent_at(handle, inode, path, lblk, flags);
}

static int
ext4_ext_max_entries(struct inode *inode, int depth)
{
        int max;

        if (depth == ext_depth(inode)) {
                if (depth == 0)
                        max = ext4_ext_space_root(inode, 1);
                else
                        max = ext4_ext_space_root_idx(inode, 1);
        } else {
                if (depth == 0)
                        max = ext4_ext_space_block(inode, 1);
                else
                        max = ext4_ext_space_block_idx(inode, 1);
        }

        return max;
}

static int ext4_valid_extent(struct inode *inode, struct ext4_extent *ext)
{
        ext4_fsblk_t block = ext4_ext_pblock(ext);
        int len = ext4_ext_get_actual_len(ext);
        ext4_lblk_t lblock = le32_to_cpu(ext->ee_block);

        /*
         * We allow neither:
         *  - zero length
         *  - overflow/wrap-around
         */
        if (lblock + len <= lblock)
                return 0;
        return ext4_inode_block_valid(inode, block, len);
}

static int ext4_valid_extent_idx(struct inode *inode,
                                struct ext4_extent_idx *ext_idx)
{
        ext4_fsblk_t block = ext4_idx_pblock(ext_idx);

        return ext4_inode_block_valid(inode, block, 1);
}

static int ext4_valid_extent_entries(struct inode *inode,
                                     struct ext4_extent_header *eh,
                                     ext4_lblk_t lblk, ext4_fsblk_t *pblk,
                                     int depth)
{
        unsigned short entries;
        ext4_lblk_t lblock = 0;
        ext4_lblk_t cur = 0;

        if (eh->eh_entries == 0)
                return 1;

        entries = le16_to_cpu(eh->eh_entries);

        if (depth == 0) {
                /* leaf entries */
                struct ext4_extent *ext = EXT_FIRST_EXTENT(eh);

                /*
                 * The logical block in the first entry should equal to
                 * the number in the index block.
                 */
                if (depth != ext_depth(inode) &&
                    lblk != le32_to_cpu(ext->ee_block))
                        return 0;
                while (entries) {
                        if (!ext4_valid_extent(inode, ext))
                                return 0;

                        /* Check for overlapping extents */
                        lblock = le32_to_cpu(ext->ee_block);
                        if (lblock < cur) {
                                *pblk = ext4_ext_pblock(ext);
                                return 0;
                        }
                        cur = lblock + ext4_ext_get_actual_len(ext);
                        ext++;
                        entries--;
                }
        } else {
                struct ext4_extent_idx *ext_idx = EXT_FIRST_INDEX(eh);

                /*
                 * The logical block in the first entry should equal to
                 * the number in the parent index block.
                 */
                if (depth != ext_depth(inode) &&
                    lblk != le32_to_cpu(ext_idx->ei_block))
                        return 0;
                while (entries) {
                        if (!ext4_valid_extent_idx(inode, ext_idx))
                                return 0;

                        /* Check for overlapping index extents */
                        lblock = le32_to_cpu(ext_idx->ei_block);
                        if (lblock < cur) {
                                *pblk = ext4_idx_pblock(ext_idx);
                                return 0;
                        }
                        ext_idx++;
                        entries--;
                        cur = lblock + 1;
                }
        }
        return 1;
}

static int __ext4_ext_check(const char *function, unsigned int line,
                            struct inode *inode, struct ext4_extent_header *eh,
                            int depth, ext4_fsblk_t pblk, ext4_lblk_t lblk)
{
        const char *error_msg;
        int max = 0, err = -EFSCORRUPTED;

        if (unlikely(eh->eh_magic != EXT4_EXT_MAGIC)) {
                error_msg = "invalid magic";
                goto corrupted;
        }
        if (unlikely(le16_to_cpu(eh->eh_depth) != depth)) {
                error_msg = "unexpected eh_depth";
                goto corrupted;
        }
        if (unlikely(eh->eh_max == 0)) {
                error_msg = "invalid eh_max";
                goto corrupted;
        }
        max = ext4_ext_max_entries(inode, depth);
        if (unlikely(le16_to_cpu(eh->eh_max) > max)) {
                error_msg = "too large eh_max";
                goto corrupted;
        }
        if (unlikely(le16_to_cpu(eh->eh_entries) > le16_to_cpu(eh->eh_max))) {
                error_msg = "invalid eh_entries";
                goto corrupted;
        }
        if (unlikely((eh->eh_entries == 0) && (depth > 0))) {
                error_msg = "eh_entries is 0 but eh_depth is > 0";
                goto corrupted;
        }
        if (!ext4_valid_extent_entries(inode, eh, lblk, &pblk, depth)) {
                error_msg = "invalid extent entries";
                goto corrupted;
        }
        if (unlikely(depth > 32)) {
                error_msg = "too large eh_depth";
                goto corrupted;
        }
        /* Verify checksum on non-root extent tree nodes */
        if (ext_depth(inode) != depth &&
            !ext4_extent_block_csum_verify(inode, eh)) {
                error_msg = "extent tree corrupted";
                err = -EFSBADCRC;
                goto corrupted;
        }
        return 0;

corrupted:
        ext4_error_inode_err(inode, function, line, 0, -err,
                             "pblk %llu bad header/extent: %s - magic %x, "
                             "entries %u, max %u(%u), depth %u(%u)",
                             (unsigned long long) pblk, error_msg,
                             le16_to_cpu(eh->eh_magic),
                             le16_to_cpu(eh->eh_entries),
                             le16_to_cpu(eh->eh_max),
                             max, le16_to_cpu(eh->eh_depth), depth);
        return err;
}

#define ext4_ext_check(inode, eh, depth, pblk)                        \
        __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk), 0)

int ext4_ext_check_inode(struct inode *inode)
{
        return ext4_ext_check(inode, ext_inode_hdr(inode), ext_depth(inode), 0);
}

static void ext4_cache_extents(struct inode *inode,
                               struct ext4_extent_header *eh)
{
        struct ext4_extent *ex = EXT_FIRST_EXTENT(eh);
        ext4_lblk_t prev = 0;
        int i;

        KUNIT_STATIC_STUB_REDIRECT(ext4_cache_extents, inode, eh);

        for (i = le16_to_cpu(eh->eh_entries); i > 0; i--, ex++) {
                unsigned int status = EXTENT_STATUS_WRITTEN;
                ext4_lblk_t lblk = le32_to_cpu(ex->ee_block);
                int len = ext4_ext_get_actual_len(ex);

                if (prev && (prev != lblk))
                        ext4_es_cache_extent(inode, prev, lblk - prev, ~0,
                                             EXTENT_STATUS_HOLE);

                if (ext4_ext_is_unwritten(ex))
                        status = EXTENT_STATUS_UNWRITTEN;
                ext4_es_cache_extent(inode, lblk, len,
                                     ext4_ext_pblock(ex), status);
                prev = lblk + len;
        }
}

static struct buffer_head *
__read_extent_tree_block(const char *function, unsigned int line,
                         struct inode *inode, struct ext4_extent_idx *idx,
                         int depth, int flags)
{
        struct buffer_head                *bh;
        int                                err;
        gfp_t                                gfp_flags = __GFP_MOVABLE | GFP_NOFS;
        ext4_fsblk_t                        pblk;

        if (flags & EXT4_EX_NOFAIL)
                gfp_flags |= __GFP_NOFAIL;

        pblk = ext4_idx_pblock(idx);
        bh = sb_getblk_gfp(inode->i_sb, pblk, gfp_flags);
        if (unlikely(!bh))
                return ERR_PTR(-ENOMEM);

        if (!bh_uptodate_or_lock(bh)) {
                trace_ext4_ext_load_extent(inode, pblk, _RET_IP_);
                err = ext4_read_bh(bh, 0, NULL, false);
                if (err < 0)
                        goto errout;
        }
        if (buffer_verified(bh) && !(flags & EXT4_EX_FORCE_CACHE))
                return bh;
        err = __ext4_ext_check(function, line, inode, ext_block_hdr(bh),
                               depth, pblk, le32_to_cpu(idx->ei_block));
        if (err)
                goto errout;
        set_buffer_verified(bh);
        /*
         * If this is a leaf block, cache all of its entries
         */
        if (!(flags & EXT4_EX_NOCACHE) && depth == 0) {
                struct ext4_extent_header *eh = ext_block_hdr(bh);
                ext4_cache_extents(inode, eh);
        }
        return bh;
errout:
        put_bh(bh);
        return ERR_PTR(err);

}

#define read_extent_tree_block(inode, idx, depth, flags)                \
        __read_extent_tree_block(__func__, __LINE__, (inode), (idx),        \
                                 (depth), (flags))

/*
 * This function is called to cache a file's extent information in the
 * extent status tree
 */
int ext4_ext_precache(struct inode *inode)
{
        struct ext4_inode_info *ei = EXT4_I(inode);
        struct ext4_ext_path *path = NULL;
        struct buffer_head *bh;
        int i = 0, depth, ret = 0;

        if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
                return 0;        /* not an extent-mapped inode */

        ext4_check_map_extents_env(inode);

        down_read(&ei->i_data_sem);
        depth = ext_depth(inode);

        /* Don't cache anything if there are no external extent blocks */
        if (!depth) {
                up_read(&ei->i_data_sem);
                return ret;
        }

        path = kzalloc_objs(struct ext4_ext_path, depth + 1, GFP_NOFS);
        if (path == NULL) {
                up_read(&ei->i_data_sem);
                return -ENOMEM;
        }

        path[0].p_hdr = ext_inode_hdr(inode);
        ret = ext4_ext_check(inode, path[0].p_hdr, depth, 0);
        if (ret)
                goto out;
        path[0].p_idx = EXT_FIRST_INDEX(path[0].p_hdr);
        while (i >= 0) {
                /*
                 * If this is a leaf block or we've reached the end of
                 * the index block, go up
                 */
                if ((i == depth) ||
                    path[i].p_idx > EXT_LAST_INDEX(path[i].p_hdr)) {
                        ext4_ext_path_brelse(path + i);
                        i--;
                        continue;
                }
                bh = read_extent_tree_block(inode, path[i].p_idx++,
                                            depth - i - 1,
                                            EXT4_EX_FORCE_CACHE);
                if (IS_ERR(bh)) {
                        ret = PTR_ERR(bh);
                        break;
                }
                i++;
                path[i].p_bh = bh;
                path[i].p_hdr = ext_block_hdr(bh);
                path[i].p_idx = EXT_FIRST_INDEX(path[i].p_hdr);
        }
        ext4_set_inode_state(inode, EXT4_STATE_EXT_PRECACHED);
out:
        up_read(&ei->i_data_sem);
        ext4_free_ext_path(path);
        return ret;
}

#ifdef EXT_DEBUG
static void ext4_ext_show_path(struct inode *inode, struct ext4_ext_path *path)
{
        int k, l = path->p_depth;

        ext_debug(inode, "path:");
        for (k = 0; k <= l; k++, path++) {
                if (path->p_idx) {
                        ext_debug(inode, "  %d->%llu",
                                  le32_to_cpu(path->p_idx->ei_block),
                                  ext4_idx_pblock(path->p_idx));
                } else if (path->p_ext) {
                        ext_debug(inode, "  %d:[%d]%d:%llu ",
                                  le32_to_cpu(path->p_ext->ee_block),
                                  ext4_ext_is_unwritten(path->p_ext),
                                  ext4_ext_get_actual_len(path->p_ext),
                                  ext4_ext_pblock(path->p_ext));
                } else
                        ext_debug(inode, "  []");
        }
        ext_debug(inode, "\n");
}

static void ext4_ext_show_leaf(struct inode *inode, struct ext4_ext_path *path)
{
        int depth = ext_depth(inode);
        struct ext4_extent_header *eh;
        struct ext4_extent *ex;
        int i;

        if (IS_ERR_OR_NULL(path))
                return;

        eh = path[depth].p_hdr;
        ex = EXT_FIRST_EXTENT(eh);

        ext_debug(inode, "Displaying leaf extents\n");

        for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ex++) {
                ext_debug(inode, "%d:[%d]%d:%llu ", le32_to_cpu(ex->ee_block),
                          ext4_ext_is_unwritten(ex),
                          ext4_ext_get_actual_len(ex), ext4_ext_pblock(ex));
        }
        ext_debug(inode, "\n");
}

static void ext4_ext_show_move(struct inode *inode, struct ext4_ext_path *path,
                        ext4_fsblk_t newblock, int level)
{
        int depth = ext_depth(inode);
        struct ext4_extent *ex;

        if (depth != level) {
                struct ext4_extent_idx *idx;
                idx = path[level].p_idx;
                while (idx <= EXT_MAX_INDEX(path[level].p_hdr)) {
                        ext_debug(inode, "%d: move %d:%llu in new index %llu\n",
                                  level, le32_to_cpu(idx->ei_block),
                                  ext4_idx_pblock(idx), newblock);
                        idx++;
                }

                return;
        }

        ex = path[depth].p_ext;
        while (ex <= EXT_MAX_EXTENT(path[depth].p_hdr)) {
                ext_debug(inode, "move %d:%llu:[%d]%d in new leaf %llu\n",
                                le32_to_cpu(ex->ee_block),
                                ext4_ext_pblock(ex),
                                ext4_ext_is_unwritten(ex),
                                ext4_ext_get_actual_len(ex),
                                newblock);
                ex++;
        }
}

#else
#define ext4_ext_show_path(inode, path)
#define ext4_ext_show_leaf(inode, path)
#define ext4_ext_show_move(inode, path, newblock, level)
#endif

/*
 * ext4_ext_binsearch_idx:
 * binary search for the closest index of the given block
 * the header must be checked before calling this
 */
static void
ext4_ext_binsearch_idx(struct inode *inode,
                        struct ext4_ext_path *path, ext4_lblk_t block)
{
        struct ext4_extent_header *eh = path->p_hdr;
        struct ext4_extent_idx *r, *l, *m;


        ext_debug(inode, "binsearch for %u(idx):  ", block);

        l = EXT_FIRST_INDEX(eh) + 1;
        r = EXT_LAST_INDEX(eh);
        while (l <= r) {
                m = l + (r - l) / 2;
                ext_debug(inode, "%p(%u):%p(%u):%p(%u) ", l,
                          le32_to_cpu(l->ei_block), m, le32_to_cpu(m->ei_block),
                          r, le32_to_cpu(r->ei_block));

                if (block < le32_to_cpu(m->ei_block))
                        r = m - 1;
                else
                        l = m + 1;
        }

        path->p_idx = l - 1;
        ext_debug(inode, "  -> %u->%lld ", le32_to_cpu(path->p_idx->ei_block),
                  ext4_idx_pblock(path->p_idx));

#ifdef CHECK_BINSEARCH
        {
                struct ext4_extent_idx *chix, *ix;
                int k;

                chix = ix = EXT_FIRST_INDEX(eh);
                for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ix++) {
                        if (k != 0 && le32_to_cpu(ix->ei_block) <=
                            le32_to_cpu(ix[-1].ei_block)) {
                                printk(KERN_DEBUG "k=%d, ix=0x%p, "
                                       "first=0x%p\n", k,
                                       ix, EXT_FIRST_INDEX(eh));
                                printk(KERN_DEBUG "%u <= %u\n",
                                       le32_to_cpu(ix->ei_block),
                                       le32_to_cpu(ix[-1].ei_block));
                        }
                        BUG_ON(k && le32_to_cpu(ix->ei_block)
                                           <= le32_to_cpu(ix[-1].ei_block));
                        if (block < le32_to_cpu(ix->ei_block))
                                break;
                        chix = ix;
                }
                BUG_ON(chix != path->p_idx);
        }
#endif

}

/*
 * ext4_ext_binsearch:
 * binary search for closest extent of the given block
 * the header must be checked before calling this
 */
static void
ext4_ext_binsearch(struct inode *inode,
                struct ext4_ext_path *path, ext4_lblk_t block)
{
        struct ext4_extent_header *eh = path->p_hdr;
        struct ext4_extent *r, *l, *m;

        if (eh->eh_entries == 0) {
                /*
                 * this leaf is empty:
                 * we get such a leaf in split/add case
                 */
                return;
        }

        ext_debug(inode, "binsearch for %u:  ", block);

        l = EXT_FIRST_EXTENT(eh) + 1;
        r = EXT_LAST_EXTENT(eh);

        while (l <= r) {
                m = l + (r - l) / 2;
                ext_debug(inode, "%p(%u):%p(%u):%p(%u) ", l,
                          le32_to_cpu(l->ee_block), m, le32_to_cpu(m->ee_block),
                          r, le32_to_cpu(r->ee_block));

                if (block < le32_to_cpu(m->ee_block))
                        r = m - 1;
                else
                        l = m + 1;
        }

        path->p_ext = l - 1;
        ext_debug(inode, "  -> %d:%llu:[%d]%d ",
                        le32_to_cpu(path->p_ext->ee_block),
                        ext4_ext_pblock(path->p_ext),
                        ext4_ext_is_unwritten(path->p_ext),
                        ext4_ext_get_actual_len(path->p_ext));

#ifdef CHECK_BINSEARCH
        {
                struct ext4_extent *chex, *ex;
                int k;

                chex = ex = EXT_FIRST_EXTENT(eh);
                for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ex++) {
                        BUG_ON(k && le32_to_cpu(ex->ee_block)
                                          <= le32_to_cpu(ex[-1].ee_block));
                        if (block < le32_to_cpu(ex->ee_block))
                                break;
                        chex = ex;
                }
                BUG_ON(chex != path->p_ext);
        }
#endif

}

void ext4_ext_tree_init(handle_t *handle, struct inode *inode)
{
        struct ext4_extent_header *eh;

        eh = ext_inode_hdr(inode);
        eh->eh_depth = 0;
        eh->eh_entries = 0;
        eh->eh_magic = EXT4_EXT_MAGIC;
        eh->eh_max = cpu_to_le16(ext4_ext_space_root(inode, 0));
        eh->eh_generation = 0;
        ext4_mark_inode_dirty(handle, inode);
}

struct ext4_ext_path *
ext4_find_extent(struct inode *inode, ext4_lblk_t block,
                 struct ext4_ext_path *path, int flags)
{
        struct ext4_extent_header *eh;
        struct buffer_head *bh;
        short int depth, i, ppos = 0;
        int ret;
        gfp_t gfp_flags = GFP_NOFS;

        KUNIT_STATIC_STUB_REDIRECT(ext4_find_extent, inode, block, path, flags);

        if (flags & EXT4_EX_NOFAIL)
                gfp_flags |= __GFP_NOFAIL;

        eh = ext_inode_hdr(inode);
        depth = ext_depth(inode);
        if (depth < 0 || depth > EXT4_MAX_EXTENT_DEPTH) {
                EXT4_ERROR_INODE(inode, "inode has invalid extent depth: %d",
                                 depth);
                ret = -EFSCORRUPTED;
                goto err;
        }

        if (path) {
                ext4_ext_drop_refs(path);
                if (depth > path[0].p_maxdepth) {
                        kfree(path);
                        path = NULL;
                }
        }
        if (!path) {
                /* account possible depth increase */
                path = kzalloc_objs(struct ext4_ext_path, depth + 2, gfp_flags);
                if (unlikely(!path))
                        return ERR_PTR(-ENOMEM);
                path[0].p_maxdepth = depth + 1;
        }
        path[0].p_hdr = eh;
        path[0].p_bh = NULL;

        i = depth;
        if (!(flags & EXT4_EX_NOCACHE) && depth == 0)
                ext4_cache_extents(inode, eh);
        /* walk through the tree */
        while (i) {
                ext_debug(inode, "depth %d: num %d, max %d\n",
                          ppos, le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));

                ext4_ext_binsearch_idx(inode, path + ppos, block);
                path[ppos].p_block = ext4_idx_pblock(path[ppos].p_idx);
                path[ppos].p_depth = i;
                path[ppos].p_ext = NULL;

                bh = read_extent_tree_block(inode, path[ppos].p_idx, --i, flags);
                if (IS_ERR(bh)) {
                        ret = PTR_ERR(bh);
                        goto err;
                }

                eh = ext_block_hdr(bh);
                ppos++;
                path[ppos].p_bh = bh;
                path[ppos].p_hdr = eh;
        }

        path[ppos].p_depth = i;
        path[ppos].p_ext = NULL;
        path[ppos].p_idx = NULL;

        /* find extent */
        ext4_ext_binsearch(inode, path + ppos, block);
        /* if not an empty leaf */
        if (path[ppos].p_ext)
                path[ppos].p_block = ext4_ext_pblock(path[ppos].p_ext);

        ext4_ext_show_path(inode, path);

        return path;

err:
        ext4_free_ext_path(path);
        return ERR_PTR(ret);
}

/*
 * ext4_ext_insert_index:
 * insert new index [@logical;@ptr] into the block at @curp;
 * check where to insert: before @curp or after @curp
 */
static int ext4_ext_insert_index(handle_t *handle, struct inode *inode,
                                 struct ext4_ext_path *curp,
                                 int logical, ext4_fsblk_t ptr)
{
        struct ext4_extent_idx *ix;
        int len, err;

        err = ext4_ext_get_access(handle, inode, curp);
        if (err)
                return err;

        if (unlikely(logical == le32_to_cpu(curp->p_idx->ei_block))) {
                EXT4_ERROR_INODE(inode,
                                 "logical %d == ei_block %d!",
                                 logical, le32_to_cpu(curp->p_idx->ei_block));
                return -EFSCORRUPTED;
        }

        if (unlikely(le16_to_cpu(curp->p_hdr->eh_entries)
                             >= le16_to_cpu(curp->p_hdr->eh_max))) {
                EXT4_ERROR_INODE(inode,
                                 "eh_entries %d >= eh_max %d!",
                                 le16_to_cpu(curp->p_hdr->eh_entries),
                                 le16_to_cpu(curp->p_hdr->eh_max));
                return -EFSCORRUPTED;
        }

        if (logical > le32_to_cpu(curp->p_idx->ei_block)) {
                /* insert after */
                ext_debug(inode, "insert new index %d after: %llu\n",
                          logical, ptr);
                ix = curp->p_idx + 1;
        } else {
                /* insert before */
                ext_debug(inode, "insert new index %d before: %llu\n",
                          logical, ptr);
                ix = curp->p_idx;
        }

        if (unlikely(ix > EXT_MAX_INDEX(curp->p_hdr))) {
                EXT4_ERROR_INODE(inode, "ix > EXT_MAX_INDEX!");
                return -EFSCORRUPTED;
        }

        len = EXT_LAST_INDEX(curp->p_hdr) - ix + 1;
        BUG_ON(len < 0);
        if (len > 0) {
                ext_debug(inode, "insert new index %d: "
                                "move %d indices from 0x%p to 0x%p\n",
                                logical, len, ix, ix + 1);
                memmove(ix + 1, ix, len * sizeof(struct ext4_extent_idx));
        }

        ix->ei_block = cpu_to_le32(logical);
        ext4_idx_store_pblock(ix, ptr);
        le16_add_cpu(&curp->p_hdr->eh_entries, 1);

        if (unlikely(ix > EXT_LAST_INDEX(curp->p_hdr))) {
                EXT4_ERROR_INODE(inode, "ix > EXT_LAST_INDEX!");
                return -EFSCORRUPTED;
        }

        err = ext4_ext_dirty(handle, inode, curp);
        ext4_std_error(inode->i_sb, err);

        return err;
}

/*
 * ext4_ext_split:
 * inserts new subtree into the path, using free index entry
 * at depth @at:
 * - allocates all needed blocks (new leaf and all intermediate index blocks)
 * - makes decision where to split
 * - moves remaining extents and index entries (right to the split point)
 *   into the newly allocated blocks
 * - initializes subtree
 */
static int ext4_ext_split(handle_t *handle, struct inode *inode,
                          unsigned int flags,
                          struct ext4_ext_path *path,
                          struct ext4_extent *newext, int at)
{
        struct buffer_head *bh = NULL;
        int depth = ext_depth(inode);
        struct ext4_extent_header *neh;
        struct ext4_extent_idx *fidx;
        int i = at, k, m, a;
        ext4_fsblk_t newblock, oldblock;
        __le32 border;
        ext4_fsblk_t *ablocks = NULL; /* array of allocated blocks */
        gfp_t gfp_flags = GFP_NOFS;
        int err = 0;
        size_t ext_size = 0;

        if (flags & EXT4_EX_NOFAIL)
                gfp_flags |= __GFP_NOFAIL;

        /* make decision: where to split? */
        /* FIXME: now decision is simplest: at current extent */

        /* if current leaf will be split, then we should use
         * border from split point */
        if (unlikely(path[depth].p_ext > EXT_MAX_EXTENT(path[depth].p_hdr))) {
                EXT4_ERROR_INODE(inode, "p_ext > EXT_MAX_EXTENT!");
                return -EFSCORRUPTED;
        }
        if (path[depth].p_ext != EXT_MAX_EXTENT(path[depth].p_hdr)) {
                border = path[depth].p_ext[1].ee_block;
                ext_debug(inode, "leaf will be split."
                                " next leaf starts at %d\n",
                                  le32_to_cpu(border));
        } else {
                border = newext->ee_block;
                ext_debug(inode, "leaf will be added."
                                " next leaf starts at %d\n",
                                le32_to_cpu(border));
        }

        /*
         * If error occurs, then we break processing
         * and mark filesystem read-only. index won't
         * be inserted and tree will be in consistent
         * state. Next mount will repair buffers too.
         */

        /*
         * Get array to track all allocated blocks.
         * We need this to handle errors and free blocks
         * upon them.
         */
        ablocks = kzalloc_objs(ext4_fsblk_t, depth, gfp_flags);
        if (!ablocks)
                return -ENOMEM;

        /* allocate all needed blocks */
        ext_debug(inode, "allocate %d blocks for indexes/leaf\n", depth - at);
        for (a = 0; a < depth - at; a++) {
                newblock = ext4_ext_new_meta_block(handle, inode, path,
                                                   newext, &err, flags);
                if (newblock == 0)
                        goto cleanup;
                ablocks[a] = newblock;
        }

        /* initialize new leaf */
        newblock = ablocks[--a];
        if (unlikely(newblock == 0)) {
                EXT4_ERROR_INODE(inode, "newblock == 0!");
                err = -EFSCORRUPTED;
                goto cleanup;
        }
        bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS);
        if (unlikely(!bh)) {
                err = -ENOMEM;
                goto cleanup;
        }
        lock_buffer(bh);

        err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
                                             EXT4_JTR_NONE);
        if (err)
                goto cleanup;

        neh = ext_block_hdr(bh);
        neh->eh_entries = 0;
        neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
        neh->eh_magic = EXT4_EXT_MAGIC;
        neh->eh_depth = 0;
        neh->eh_generation = 0;

        /* move remainder of path[depth] to the new leaf */
        if (unlikely(path[depth].p_hdr->eh_entries !=
                     path[depth].p_hdr->eh_max)) {
                EXT4_ERROR_INODE(inode, "eh_entries %d != eh_max %d!",
                                 path[depth].p_hdr->eh_entries,
                                 path[depth].p_hdr->eh_max);
                err = -EFSCORRUPTED;
                goto cleanup;
        }
        /* start copy from next extent */
        m = EXT_MAX_EXTENT(path[depth].p_hdr) - path[depth].p_ext++;
        ext4_ext_show_move(inode, path, newblock, depth);
        if (m) {
                struct ext4_extent *ex;
                ex = EXT_FIRST_EXTENT(neh);
                memmove(ex, path[depth].p_ext, sizeof(struct ext4_extent) * m);
                le16_add_cpu(&neh->eh_entries, m);
        }

        /* zero out unused area in the extent block */
        ext_size = sizeof(struct ext4_extent_header) +
                sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries);
        memset(bh->b_data + ext_size, 0, inode->i_sb->s_blocksize - ext_size);
        ext4_extent_block_csum_set(inode, neh);
        set_buffer_uptodate(bh);
        unlock_buffer(bh);

        err = ext4_handle_dirty_metadata(handle, inode, bh);
        if (err)
                goto cleanup;
        brelse(bh);
        bh = NULL;

        /* correct old leaf */
        if (m) {
                err = ext4_ext_get_access(handle, inode, path + depth);
                if (err)
                        goto cleanup;
                le16_add_cpu(&path[depth].p_hdr->eh_entries, -m);
                err = ext4_ext_dirty(handle, inode, path + depth);
                if (err)
                        goto cleanup;

        }

        /* create intermediate indexes */
        k = depth - at - 1;
        if (unlikely(k < 0)) {
                EXT4_ERROR_INODE(inode, "k %d < 0!", k);
                err = -EFSCORRUPTED;
                goto cleanup;
        }
        if (k)
                ext_debug(inode, "create %d intermediate indices\n", k);
        /* insert new index into current index block */
        /* current depth stored in i var */
        i = depth - 1;
        while (k--) {
                oldblock = newblock;
                newblock = ablocks[--a];
                bh = sb_getblk(inode->i_sb, newblock);
                if (unlikely(!bh)) {
                        err = -ENOMEM;
                        goto cleanup;
                }
                lock_buffer(bh);

                err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
                                                     EXT4_JTR_NONE);
                if (err)
                        goto cleanup;

                neh = ext_block_hdr(bh);
                neh->eh_entries = cpu_to_le16(1);
                neh->eh_magic = EXT4_EXT_MAGIC;
                neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
                neh->eh_depth = cpu_to_le16(depth - i);
                neh->eh_generation = 0;
                fidx = EXT_FIRST_INDEX(neh);
                fidx->ei_block = border;
                ext4_idx_store_pblock(fidx, oldblock);

                ext_debug(inode, "int.index at %d (block %llu): %u -> %llu\n",
                                i, newblock, le32_to_cpu(border), oldblock);

                /* move remainder of path[i] to the new index block */
                if (unlikely(EXT_MAX_INDEX(path[i].p_hdr) !=
                                        EXT_LAST_INDEX(path[i].p_hdr))) {
                        EXT4_ERROR_INODE(inode,
                                         "EXT_MAX_INDEX != EXT_LAST_INDEX ee_block %d!",
                                         le32_to_cpu(path[i].p_ext->ee_block));
                        err = -EFSCORRUPTED;
                        goto cleanup;
                }
                /* start copy indexes */
                m = EXT_MAX_INDEX(path[i].p_hdr) - path[i].p_idx++;
                ext_debug(inode, "cur 0x%p, last 0x%p\n", path[i].p_idx,
                                EXT_MAX_INDEX(path[i].p_hdr));
                ext4_ext_show_move(inode, path, newblock, i);
                if (m) {
                        memmove(++fidx, path[i].p_idx,
                                sizeof(struct ext4_extent_idx) * m);
                        le16_add_cpu(&neh->eh_entries, m);
                }
                /* zero out unused area in the extent block */
                ext_size = sizeof(struct ext4_extent_header) +
                   (sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries));
                memset(bh->b_data + ext_size, 0,
                        inode->i_sb->s_blocksize - ext_size);
                ext4_extent_block_csum_set(inode, neh);
                set_buffer_uptodate(bh);
                unlock_buffer(bh);

                err = ext4_handle_dirty_metadata(handle, inode, bh);
                if (err)
                        goto cleanup;
                brelse(bh);
                bh = NULL;

                /* correct old index */
                if (m) {
                        err = ext4_ext_get_access(handle, inode, path + i);
                        if (err)
                                goto cleanup;
                        le16_add_cpu(&path[i].p_hdr->eh_entries, -m);
                        err = ext4_ext_dirty(handle, inode, path + i);
                        if (err)
                                goto cleanup;
                }

                i--;
        }

        /* insert new index */
        err = ext4_ext_insert_index(handle, inode, path + at,
                                    le32_to_cpu(border), newblock);

cleanup:
        if (bh) {
                if (buffer_locked(bh))
                        unlock_buffer(bh);
                brelse(bh);
        }

        if (err) {
                /* free all allocated blocks in error case */
                for (i = 0; i < depth; i++) {
                        if (!ablocks[i])
                                continue;
                        ext4_free_blocks(handle, inode, NULL, ablocks[i], 1,
                                         EXT4_FREE_BLOCKS_METADATA);
                }
        }
        kfree(ablocks);

        return err;
}

/*
 * ext4_ext_grow_indepth:
 * implements tree growing procedure:
 * - allocates new block
 * - moves top-level data (index block or leaf) into the new block
 * - initializes new top-level, creating index that points to the
 *   just created block
 */
static int ext4_ext_grow_indepth(handle_t *handle, struct inode *inode,
                                 unsigned int flags)
{
        struct ext4_extent_header *neh;
        struct buffer_head *bh;
        ext4_fsblk_t newblock, goal = 0;
        struct ext4_super_block *es = EXT4_SB(inode->i_sb)->s_es;
        int err = 0;
        size_t ext_size = 0;

        /* Try to prepend new index to old one */
        if (ext_depth(inode))
                goal = ext4_idx_pblock(EXT_FIRST_INDEX(ext_inode_hdr(inode)));
        if (goal > le32_to_cpu(es->s_first_data_block)) {
                flags |= EXT4_MB_HINT_TRY_GOAL;
                goal--;
        } else
                goal = ext4_inode_to_goal_block(inode);
        newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
                                        NULL, &err);
        if (newblock == 0)
                return err;

        bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS);
        if (unlikely(!bh))
                return -ENOMEM;
        lock_buffer(bh);

        err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
                                             EXT4_JTR_NONE);
        if (err) {
                unlock_buffer(bh);
                goto out;
        }

        ext_size = sizeof(EXT4_I(inode)->i_data);
        /* move top-level index/leaf into new block */
        memmove(bh->b_data, EXT4_I(inode)->i_data, ext_size);
        /* zero out unused area in the extent block */
        memset(bh->b_data + ext_size, 0, inode->i_sb->s_blocksize - ext_size);

        /* set size of new block */
        neh = ext_block_hdr(bh);
        /* old root could have indexes or leaves
         * so calculate e_max right way */
        if (ext_depth(inode))
                neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
        else
                neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
        neh->eh_magic = EXT4_EXT_MAGIC;
        ext4_extent_block_csum_set(inode, neh);
        set_buffer_uptodate(bh);
        set_buffer_verified(bh);
        unlock_buffer(bh);

        err = ext4_handle_dirty_metadata(handle, inode, bh);
        if (err)
                goto out;

        /* Update top-level index: num,max,pointer */
        neh = ext_inode_hdr(inode);
        neh->eh_entries = cpu_to_le16(1);
        ext4_idx_store_pblock(EXT_FIRST_INDEX(neh), newblock);
        if (neh->eh_depth == 0) {
                /* Root extent block becomes index block */
                neh->eh_max = cpu_to_le16(ext4_ext_space_root_idx(inode, 0));
                EXT_FIRST_INDEX(neh)->ei_block =
                        EXT_FIRST_EXTENT(neh)->ee_block;
        }
        ext_debug(inode, "new root: num %d(%d), lblock %d, ptr %llu\n",
                  le16_to_cpu(neh->eh_entries), le16_to_cpu(neh->eh_max),
                  le32_to_cpu(EXT_FIRST_INDEX(neh)->ei_block),
                  ext4_idx_pblock(EXT_FIRST_INDEX(neh)));

        le16_add_cpu(&neh->eh_depth, 1);
        err = ext4_mark_inode_dirty(handle, inode);
out:
        brelse(bh);

        return err;
}

/*
 * ext4_ext_create_new_leaf:
 * finds empty index and adds new leaf.
 * if no free index is found, then it requests in-depth growing.
 */
static struct ext4_ext_path *
ext4_ext_create_new_leaf(handle_t *handle, struct inode *inode,
                         unsigned int mb_flags, unsigned int gb_flags,
                         struct ext4_ext_path *path,
                         struct ext4_extent *newext)
{
        struct ext4_ext_path *curp;
        int depth, i, err = 0;
        ext4_lblk_t ee_block = le32_to_cpu(newext->ee_block);

repeat:
        i = depth = ext_depth(inode);

        /* walk up to the tree and look for free index entry */
        curp = path + depth;
        while (i > 0 && !EXT_HAS_FREE_INDEX(curp)) {
                i--;
                curp--;
        }

        /* we use already allocated block for index block,
         * so subsequent data blocks should be contiguous */
        if (EXT_HAS_FREE_INDEX(curp)) {
                /* if we found index with free entry, then use that
                 * entry: create all needed subtree and add new leaf */
                err = ext4_ext_split(handle, inode, mb_flags, path, newext, i);
                if (err)
                        goto errout;

                /* refill path */
                path = ext4_find_extent(inode, ee_block, path, gb_flags);
                return path;
        }

        /* tree is full, time to grow in depth */
        err = ext4_ext_grow_indepth(handle, inode, mb_flags);
        if (err)
                goto errout;

        /* refill path */
        path = ext4_find_extent(inode, ee_block, path, gb_flags);
        if (IS_ERR(path))
                return path;

        /*
         * only first (depth 0 -> 1) produces free space;
         * in all other cases we have to split the grown tree
         */
        depth = ext_depth(inode);
        if (path[depth].p_hdr->eh_entries == path[depth].p_hdr->eh_max) {
                /* now we need to split */
                goto repeat;
        }

        return path;

errout:
        ext4_free_ext_path(path);
        return ERR_PTR(err);
}

/*
 * search the closest allocated block to the left for *logical
 * and returns it at @logical + it's physical address at @phys
 * if *logical is the smallest allocated block, the function
 * returns 0 at @phys
 * return value contains 0 (success) or error code
 */
static int ext4_ext_search_left(struct inode *inode,
                                struct ext4_ext_path *path,
                                ext4_lblk_t *logical, ext4_fsblk_t *phys)
{
        struct ext4_extent_idx *ix;
        struct ext4_extent *ex;
        int depth, ee_len;

        if (unlikely(path == NULL)) {
                EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
                return -EFSCORRUPTED;
        }
        depth = path->p_depth;
        *phys = 0;

        if (depth == 0 && path->p_ext == NULL)
                return 0;

        /* usually extent in the path covers blocks smaller
         * then *logical, but it can be that extent is the
         * first one in the file */

        ex = path[depth].p_ext;
        ee_len = ext4_ext_get_actual_len(ex);
        if (*logical < le32_to_cpu(ex->ee_block)) {
                if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
                        EXT4_ERROR_INODE(inode,
                                         "EXT_FIRST_EXTENT != ex *logical %d ee_block %d!",
                                         *logical, le32_to_cpu(ex->ee_block));
                        return -EFSCORRUPTED;
                }
                while (--depth >= 0) {
                        ix = path[depth].p_idx;
                        if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
                                EXT4_ERROR_INODE(inode,
                                  "ix (%d) != EXT_FIRST_INDEX (%d) (depth %d)!",
                                  ix != NULL ? le32_to_cpu(ix->ei_block) : 0,
                                  le32_to_cpu(EXT_FIRST_INDEX(path[depth].p_hdr)->ei_block),
                                  depth);
                                return -EFSCORRUPTED;
                        }
                }
                return 0;
        }

        if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
                EXT4_ERROR_INODE(inode,
                                 "logical %d < ee_block %d + ee_len %d!",
                                 *logical, le32_to_cpu(ex->ee_block), ee_len);
                return -EFSCORRUPTED;
        }

        *logical = le32_to_cpu(ex->ee_block) + ee_len - 1;
        *phys = ext4_ext_pblock(ex) + ee_len - 1;
        return 0;
}

/*
 * Search the closest allocated block to the right for *logical
 * and returns it at @logical + it's physical address at @phys.
 * If not exists, return 0 and @phys is set to 0. We will return
 * 1 which means we found an allocated block and ret_ex is valid.
 * Or return a (< 0) error code.
 */
static int ext4_ext_search_right(struct inode *inode,
                                 struct ext4_ext_path *path,
                                 ext4_lblk_t *logical, ext4_fsblk_t *phys,
                                 struct ext4_extent *ret_ex, int flags)
{
        struct buffer_head *bh = NULL;
        struct ext4_extent_header *eh;
        struct ext4_extent_idx *ix;
        struct ext4_extent *ex;
        int depth;        /* Note, NOT eh_depth; depth from top of tree */
        int ee_len;

        if (unlikely(path == NULL)) {
                EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
                return -EFSCORRUPTED;
        }
        depth = path->p_depth;
        *phys = 0;

        if (depth == 0 && path->p_ext == NULL)
                return 0;

        /* usually extent in the path covers blocks smaller
         * then *logical, but it can be that extent is the
         * first one in the file */

        ex = path[depth].p_ext;
        ee_len = ext4_ext_get_actual_len(ex);
        if (*logical < le32_to_cpu(ex->ee_block)) {
                if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
                        EXT4_ERROR_INODE(inode,
                                         "first_extent(path[%d].p_hdr) != ex",
                                         depth);
                        return -EFSCORRUPTED;
                }
                while (--depth >= 0) {
                        ix = path[depth].p_idx;
                        if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
                                EXT4_ERROR_INODE(inode,
                                                 "ix != EXT_FIRST_INDEX *logical %d!",
                                                 *logical);
                                return -EFSCORRUPTED;
                        }
                }
                goto found_extent;
        }

        if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
                EXT4_ERROR_INODE(inode,
                                 "logical %d < ee_block %d + ee_len %d!",
                                 *logical, le32_to_cpu(ex->ee_block), ee_len);
                return -EFSCORRUPTED;
        }

        if (ex != EXT_LAST_EXTENT(path[depth].p_hdr)) {
                /* next allocated block in this leaf */
                ex++;
                goto found_extent;
        }

        /* go up and search for index to the right */
        while (--depth >= 0) {
                ix = path[depth].p_idx;
                if (ix != EXT_LAST_INDEX(path[depth].p_hdr))
                        goto got_index;
        }

        /* we've gone up to the root and found no index to the right */
        return 0;

got_index:
        /* we've found index to the right, let's
         * follow it and find the closest allocated
         * block to the right */
        ix++;
        while (++depth < path->p_depth) {
                /* subtract from p_depth to get proper eh_depth */
                bh = read_extent_tree_block(inode, ix, path->p_depth - depth,
                                            flags);
                if (IS_ERR(bh))
                        return PTR_ERR(bh);
                eh = ext_block_hdr(bh);
                ix = EXT_FIRST_INDEX(eh);
                put_bh(bh);
        }

        bh = read_extent_tree_block(inode, ix, path->p_depth - depth, flags);
        if (IS_ERR(bh))
                return PTR_ERR(bh);
        eh = ext_block_hdr(bh);
        ex = EXT_FIRST_EXTENT(eh);
found_extent:
        *logical = le32_to_cpu(ex->ee_block);
        *phys = ext4_ext_pblock(ex);
        if (ret_ex)
                *ret_ex = *ex;
        if (bh)
                put_bh(bh);
        return 1;
}

/*
 * ext4_ext_next_allocated_block:
 * returns allocated block in subsequent extent or EXT_MAX_BLOCKS.
 * NOTE: it considers block number from index entry as
 * allocated block. Thus, index entries have to be consistent
 * with leaves.
 */
ext4_lblk_t
ext4_ext_next_allocated_block(struct ext4_ext_path *path)
{
        int depth;

        BUG_ON(path == NULL);
        depth = path->p_depth;

        if (depth == 0 && path->p_ext == NULL)
                return EXT_MAX_BLOCKS;

        while (depth >= 0) {
                struct ext4_ext_path *p = &path[depth];

                if (depth == path->p_depth) {
                        /* leaf */
                        if (p->p_ext && p->p_ext != EXT_LAST_EXTENT(p->p_hdr))
                                return le32_to_cpu(p->p_ext[1].ee_block);
                } else {
                        /* index */
                        if (p->p_idx != EXT_LAST_INDEX(p->p_hdr))
                                return le32_to_cpu(p->p_idx[1].ei_block);
                }
                depth--;
        }

        return EXT_MAX_BLOCKS;
}

/*
 * ext4_ext_next_leaf_block:
 * returns first allocated block from next leaf or EXT_MAX_BLOCKS
 */
static ext4_lblk_t ext4_ext_next_leaf_block(struct ext4_ext_path *path)
{
        int depth;

        BUG_ON(path == NULL);
        depth = path->p_depth;

        /* zero-tree has no leaf blocks at all */
        if (depth == 0)
                return EXT_MAX_BLOCKS;

        /* go to index block */
        depth--;

        while (depth >= 0) {
                if (path[depth].p_idx !=
                                EXT_LAST_INDEX(path[depth].p_hdr))
                        return (ext4_lblk_t)
                                le32_to_cpu(path[depth].p_idx[1].ei_block);
                depth--;
        }

        return EXT_MAX_BLOCKS;
}

/*
 * ext4_ext_correct_indexes:
 * if leaf gets modified and modified extent is first in the leaf,
 * then we have to correct all indexes above.
 * TODO: do we need to correct tree in all cases?
 */
static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode,
                                struct ext4_ext_path *path)
{
        struct ext4_extent_header *eh;
        int depth = ext_depth(inode);
        struct ext4_extent *ex;
        __le32 border;
        int k, err = 0;

        eh = path[depth].p_hdr;
        ex = path[depth].p_ext;

        if (unlikely(ex == NULL || eh == NULL)) {
                EXT4_ERROR_INODE(inode,
                                 "ex %p == NULL or eh %p == NULL", ex, eh);
                return -EFSCORRUPTED;
        }

        if (depth == 0) {
                /* there is no tree at all */
                return 0;
        }

        if (ex != EXT_FIRST_EXTENT(eh)) {
                /* we correct tree if first leaf got modified only */
                return 0;
        }

        /*
         * TODO: we need correction if border is smaller than current one
         */
        k = depth - 1;
        border = path[depth].p_ext->ee_block;
        err = ext4_ext_get_access(handle, inode, path + k);
        if (err)
                return err;
        if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) {
                EXT4_ERROR_INODE(inode,
                                 "path[%d].p_idx %p > EXT_LAST_INDEX %p",
                                 k, path[k].p_idx,
                                 EXT_LAST_INDEX(path[k].p_hdr));
                return -EFSCORRUPTED;
        }
        path[k].p_idx->ei_block = border;
        err = ext4_ext_dirty(handle, inode, path + k);
        if (err)
                return err;

        while (k--) {
                /* change all left-side indexes */
                if (path[k+1].p_idx != EXT_FIRST_INDEX(path[k+1].p_hdr))
                        break;
                err = ext4_ext_get_access(handle, inode, path + k);
                if (err)
                        goto clean;
                if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) {
                        EXT4_ERROR_INODE(inode,
                                         "path[%d].p_idx %p > EXT_LAST_INDEX %p",
                                         k, path[k].p_idx,
                                         EXT_LAST_INDEX(path[k].p_hdr));
                        err = -EFSCORRUPTED;
                        goto clean;
                }
                path[k].p_idx->ei_block = border;
                err = ext4_ext_dirty(handle, inode, path + k);
                if (err)
                        goto clean;
        }
        return 0;

clean:
        /*
         * The path[k].p_bh is either unmodified or with no verified bit
         * set (see ext4_ext_get_access()). So just clear the verified bit
         * of the successfully modified extents buffers, which will force
         * these extents to be checked to avoid using inconsistent data.
         */
        while (++k < depth)
                clear_buffer_verified(path[k].p_bh);

        return err;
}

static int ext4_can_extents_be_merged(struct inode *inode,
                                      struct ext4_extent *ex1,
                                      struct ext4_extent *ex2)
{
        unsigned short ext1_ee_len, ext2_ee_len;

        if (ext4_ext_is_unwritten(ex1) != ext4_ext_is_unwritten(ex2))
                return 0;

        ext1_ee_len = ext4_ext_get_actual_len(ex1);
        ext2_ee_len = ext4_ext_get_actual_len(ex2);

        if (le32_to_cpu(ex1->ee_block) + ext1_ee_len !=
                        le32_to_cpu(ex2->ee_block))
                return 0;

        if (ext1_ee_len + ext2_ee_len > EXT_INIT_MAX_LEN)
                return 0;

        if (ext4_ext_is_unwritten(ex1) &&
            ext1_ee_len + ext2_ee_len > EXT_UNWRITTEN_MAX_LEN)
                return 0;
#ifdef AGGRESSIVE_TEST
        if (ext1_ee_len >= 4)
                return 0;
#endif

        if (ext4_ext_pblock(ex1) + ext1_ee_len == ext4_ext_pblock(ex2))
                return 1;
        return 0;
}

/*
 * This function tries to merge the "ex" extent to the next extent in the tree.
 * It always tries to merge towards right. If you want to merge towards
 * left, pass "ex - 1" as argument instead of "ex".
 * Returns 0 if the extents (ex and ex+1) were _not_ merged and returns
 * 1 if they got merged.
 */
static int ext4_ext_try_to_merge_right(struct inode *inode,
                                 struct ext4_ext_path *path,
                                 struct ext4_extent *ex)
{
        struct ext4_extent_header *eh;
        unsigned int depth, len;
        int merge_done = 0, unwritten;

        depth = ext_depth(inode);
        BUG_ON(path[depth].p_hdr == NULL);
        eh = path[depth].p_hdr;

        while (ex < EXT_LAST_EXTENT(eh)) {
                if (!ext4_can_extents_be_merged(inode, ex, ex + 1))
                        break;
                /* merge with next extent! */
                unwritten = ext4_ext_is_unwritten(ex);
                ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
                                + ext4_ext_get_actual_len(ex + 1));
                if (unwritten)
                        ext4_ext_mark_unwritten(ex);

                if (ex + 1 < EXT_LAST_EXTENT(eh)) {
                        len = (EXT_LAST_EXTENT(eh) - ex - 1)
                                * sizeof(struct ext4_extent);
                        memmove(ex + 1, ex + 2, len);
                }
                le16_add_cpu(&eh->eh_entries, -1);
                merge_done = 1;
                WARN_ON(eh->eh_entries == 0);
                if (!eh->eh_entries)
                        EXT4_ERROR_INODE(inode, "eh->eh_entries = 0!");
        }

        return merge_done;
}

/*
 * This function does a very simple check to see if we can collapse
 * an extent tree with a single extent tree leaf block into the inode.
 */
static void ext4_ext_try_to_merge_up(handle_t *handle,
                                     struct inode *inode,
                                     struct ext4_ext_path *path)
{
        size_t s;
        unsigned max_root = ext4_ext_space_root(inode, 0);
        ext4_fsblk_t blk;

        if ((path[0].p_depth != 1) ||
            (le16_to_cpu(path[0].p_hdr->eh_entries) != 1) ||
            (le16_to_cpu(path[1].p_hdr->eh_entries) > max_root))
                return;

        /*
         * We need to modify the block allocation bitmap and the block
         * group descriptor to release the extent tree block.  If we
         * can't get the journal credits, give up.
         */
        if (ext4_journal_extend(handle, 2,
                        ext4_free_metadata_revoke_credits(inode->i_sb, 1)))
                return;

        /*
         * Copy the extent data up to the inode
         */
        blk = ext4_idx_pblock(path[0].p_idx);
        s = le16_to_cpu(path[1].p_hdr->eh_entries) *
                sizeof(struct ext4_extent_idx);
        s += sizeof(struct ext4_extent_header);

        path[1].p_maxdepth = path[0].p_maxdepth;
        memcpy(path[0].p_hdr, path[1].p_hdr, s);
        path[0].p_depth = 0;
        path[0].p_ext = EXT_FIRST_EXTENT(path[0].p_hdr) +
                (path[1].p_ext - EXT_FIRST_EXTENT(path[1].p_hdr));
        path[0].p_hdr->eh_max = cpu_to_le16(max_root);

        ext4_ext_path_brelse(path + 1);
        ext4_free_blocks(handle, inode, NULL, blk, 1,
                         EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);
}

/*
 * This function tries to merge the @ex extent to neighbours in the tree, then
 * tries to collapse the extent tree into the inode.
 */
static void ext4_ext_try_to_merge(handle_t *handle,
                                  struct inode *inode,
                                  struct ext4_ext_path *path,
                                  struct ext4_extent *ex)
{
        struct ext4_extent_header *eh;
        unsigned int depth;
        int merge_done = 0;

        depth = ext_depth(inode);
        BUG_ON(path[depth].p_hdr == NULL);
        eh = path[depth].p_hdr;

        if (ex > EXT_FIRST_EXTENT(eh))
                merge_done = ext4_ext_try_to_merge_right(inode, path, ex - 1);

        if (!merge_done)
                (void) ext4_ext_try_to_merge_right(inode, path, ex);

        ext4_ext_try_to_merge_up(handle, inode, path);
}

/*
 * check if a portion of the "newext" extent overlaps with an
 * existing extent.
 *
 * If there is an overlap discovered, it updates the length of the newext
 * such that there will be no overlap, and then returns 1.
 * If there is no overlap found, it returns 0.
 */
static unsigned int ext4_ext_check_overlap(struct ext4_sb_info *sbi,
                                           struct inode *inode,
                                           struct ext4_extent *newext,
                                           struct ext4_ext_path *path)
{
        ext4_lblk_t b1, b2;
        unsigned int depth, len1;
        unsigned int ret = 0;

        b1 = le32_to_cpu(newext->ee_block);
        len1 = ext4_ext_get_actual_len(newext);
        depth = ext_depth(inode);
        if (!path[depth].p_ext)
                goto out;
        b2 = EXT4_LBLK_CMASK(sbi, le32_to_cpu(path[depth].p_ext->ee_block));

        /*
         * get the next allocated block if the extent in the path
         * is before the requested block(s)
         */
        if (b2 < b1) {
                b2 = ext4_ext_next_allocated_block(path);
                if (b2 == EXT_MAX_BLOCKS)
                        goto out;
                b2 = EXT4_LBLK_CMASK(sbi, b2);
        }

        /* check for wrap through zero on extent logical start block*/
        if (b1 + len1 < b1) {
                len1 = EXT_MAX_BLOCKS - b1;
                newext->ee_len = cpu_to_le16(len1);
                ret = 1;
        }

        /* check for overlap */
        if (b1 + len1 > b2) {
                newext->ee_len = cpu_to_le16(b2 - b1);
                ret = 1;
        }
out:
        return ret;
}

/*
 * ext4_ext_insert_extent:
 * tries to merge requested extent into the existing extent or
 * inserts requested extent as new one into the tree,
 * creating new leaf in the no-space case.
 */
struct ext4_ext_path *
ext4_ext_insert_extent(handle_t *handle, struct inode *inode,
                       struct ext4_ext_path *path,
                       struct ext4_extent *newext, int gb_flags)
{
        struct ext4_extent_header *eh;
        struct ext4_extent *ex, *fex;
        struct ext4_extent *nearex; /* nearest extent */
        int depth, len, err = 0;
        ext4_lblk_t next;
        int mb_flags = 0, unwritten;

        KUNIT_STATIC_STUB_REDIRECT(ext4_ext_insert_extent, handle, inode, path,
                                   newext, gb_flags);

        if (gb_flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
                mb_flags |= EXT4_MB_DELALLOC_RESERVED;
        if (unlikely(ext4_ext_get_actual_len(newext) == 0)) {
                EXT4_ERROR_INODE(inode, "ext4_ext_get_actual_len(newext) == 0");
                err = -EFSCORRUPTED;
                goto errout;
        }
        depth = ext_depth(inode);
        ex = path[depth].p_ext;
        eh = path[depth].p_hdr;
        if (unlikely(path[depth].p_hdr == NULL)) {
                EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
                err = -EFSCORRUPTED;
                goto errout;
        }

        /* try to insert block into found extent and return */
        if (ex && !(gb_flags & EXT4_GET_BLOCKS_SPLIT_NOMERGE)) {

                /*
                 * Try to see whether we should rather test the extent on
                 * right from ex, or from the left of ex. This is because
                 * ext4_find_extent() can return either extent on the
                 * left, or on the right from the searched position. This
                 * will make merging more effective.
                 */
                if (ex < EXT_LAST_EXTENT(eh) &&
                    (le32_to_cpu(ex->ee_block) +
                    ext4_ext_get_actual_len(ex) <
                    le32_to_cpu(newext->ee_block))) {
                        ex += 1;
                        goto prepend;
                } else if ((ex > EXT_FIRST_EXTENT(eh)) &&
                           (le32_to_cpu(newext->ee_block) +
                           ext4_ext_get_actual_len(newext) <
                           le32_to_cpu(ex->ee_block)))
                        ex -= 1;

                /* Try to append newex to the ex */
                if (ext4_can_extents_be_merged(inode, ex, newext)) {
                        ext_debug(inode, "append [%d]%d block to %u:[%d]%d"
                                  "(from %llu)\n",
                                  ext4_ext_is_unwritten(newext),
                                  ext4_ext_get_actual_len(newext),
                                  le32_to_cpu(ex->ee_block),
                                  ext4_ext_is_unwritten(ex),
                                  ext4_ext_get_actual_len(ex),
                                  ext4_ext_pblock(ex));
                        err = ext4_ext_get_access(handle, inode,
                                                  path + depth);
                        if (err)
                                goto errout;
                        unwritten = ext4_ext_is_unwritten(ex);
                        ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
                                        + ext4_ext_get_actual_len(newext));
                        if (unwritten)
                                ext4_ext_mark_unwritten(ex);
                        nearex = ex;
                        goto merge;
                }

prepend:
                /* Try to prepend newex to the ex */
                if (ext4_can_extents_be_merged(inode, newext, ex)) {
                        ext_debug(inode, "prepend %u[%d]%d block to %u:[%d]%d"
                                  "(from %llu)\n",
                                  le32_to_cpu(newext->ee_block),
                                  ext4_ext_is_unwritten(newext),
                                  ext4_ext_get_actual_len(newext),
                                  le32_to_cpu(ex->ee_block),
                                  ext4_ext_is_unwritten(ex),
                                  ext4_ext_get_actual_len(ex),
                                  ext4_ext_pblock(ex));
                        err = ext4_ext_get_access(handle, inode,
                                                  path + depth);
                        if (err)
                                goto errout;

                        unwritten = ext4_ext_is_unwritten(ex);
                        ex->ee_block = newext->ee_block;
                        ext4_ext_store_pblock(ex, ext4_ext_pblock(newext));
                        ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
                                        + ext4_ext_get_actual_len(newext));
                        if (unwritten)
                                ext4_ext_mark_unwritten(ex);
                        nearex = ex;
                        goto merge;
                }
        }

        depth = ext_depth(inode);
        eh = path[depth].p_hdr;
        if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max))
                goto has_space;

        /* probably next leaf has space for us? */
        fex = EXT_LAST_EXTENT(eh);
        next = EXT_MAX_BLOCKS;
        if (le32_to_cpu(newext->ee_block) > le32_to_cpu(fex->ee_block))
                next = ext4_ext_next_leaf_block(path);
        if (next != EXT_MAX_BLOCKS) {
                struct ext4_ext_path *npath;

                ext_debug(inode, "next leaf block - %u\n", next);
                npath = ext4_find_extent(inode, next, NULL, gb_flags);
                if (IS_ERR(npath)) {
                        err = PTR_ERR(npath);
                        goto errout;
                }
                BUG_ON(npath->p_depth != path->p_depth);
                eh = npath[depth].p_hdr;
                if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max)) {
                        ext_debug(inode, "next leaf isn't full(%d)\n",
                                  le16_to_cpu(eh->eh_entries));
                        ext4_free_ext_path(path);
                        path = npath;
                        goto has_space;
                }
                ext_debug(inode, "next leaf has no free space(%d,%d)\n",
                          le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));
                ext4_free_ext_path(npath);
        }

        /*
         * There is no free space in the found leaf.
         * We're gonna add a new leaf in the tree.
         */
        if (gb_flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
                mb_flags |= EXT4_MB_USE_RESERVED;
        path = ext4_ext_create_new_leaf(handle, inode, mb_flags, gb_flags,
                                        path, newext);
        if (IS_ERR(path))
                return path;
        depth = ext_depth(inode);
        eh = path[depth].p_hdr;

has_space:
        nearex = path[depth].p_ext;

        err = ext4_ext_get_access(handle, inode, path + depth);
        if (err)
                goto errout;

        if (!nearex) {
                /* there is no extent in this leaf, create first one */
                ext_debug(inode, "first extent in the leaf: %u:%llu:[%d]%d\n",
                                le32_to_cpu(newext->ee_block),
                                ext4_ext_pblock(newext),
                                ext4_ext_is_unwritten(newext),
                                ext4_ext_get_actual_len(newext));
                nearex = EXT_FIRST_EXTENT(eh);
        } else {
                if (le32_to_cpu(newext->ee_block)
                           > le32_to_cpu(nearex->ee_block)) {
                        /* Insert after */
                        ext_debug(inode, "insert %u:%llu:[%d]%d before: "
                                        "nearest %p\n",
                                        le32_to_cpu(newext->ee_block),
                                        ext4_ext_pblock(newext),
                                        ext4_ext_is_unwritten(newext),
                                        ext4_ext_get_actual_len(newext),
                                        nearex);
                        nearex++;
                } else {
                        /* Insert before */
                        BUG_ON(newext->ee_block == nearex->ee_block);
                        ext_debug(inode, "insert %u:%llu:[%d]%d after: "
                                        "nearest %p\n",
                                        le32_to_cpu(newext->ee_block),
                                        ext4_ext_pblock(newext),
                                        ext4_ext_is_unwritten(newext),
                                        ext4_ext_get_actual_len(newext),
                                        nearex);
                }
                len = EXT_LAST_EXTENT(eh) - nearex + 1;
                if (len > 0) {
                        ext_debug(inode, "insert %u:%llu:[%d]%d: "
                                        "move %d extents from 0x%p to 0x%p\n",
                                        le32_to_cpu(newext->ee_block),
                                        ext4_ext_pblock(newext),
                                        ext4_ext_is_unwritten(newext),
                                        ext4_ext_get_actual_len(newext),
                                        len, nearex, nearex + 1);
                        memmove(nearex + 1, nearex,
                                len * sizeof(struct ext4_extent));
                }
        }

        le16_add_cpu(&eh->eh_entries, 1);
        path[depth].p_ext = nearex;
        nearex->ee_block = newext->ee_block;
        ext4_ext_store_pblock(nearex, ext4_ext_pblock(newext));
        nearex->ee_len = newext->ee_len;

merge:
        /* try to merge extents */
        if (!(gb_flags & EXT4_GET_BLOCKS_SPLIT_NOMERGE))
                ext4_ext_try_to_merge(handle, inode, path, nearex);

        /* time to correct all indexes above */
        err = ext4_ext_correct_indexes(handle, inode, path);
        if (err)
                goto errout;

        err = ext4_ext_dirty(handle, inode, path + path->p_depth);
        if (err)
                goto errout;

        return path;

errout:
        ext4_free_ext_path(path);
        return ERR_PTR(err);
}

static int ext4_fill_es_cache_info(struct inode *inode,
                                   ext4_lblk_t block, ext4_lblk_t num,
                                   struct fiemap_extent_info *fieinfo)
{
        ext4_lblk_t next, end = block + num - 1;
        struct extent_status es;
        unsigned char blksize_bits = inode->i_sb->s_blocksize_bits;
        unsigned int flags;
        int err;

        while (block <= end) {
                next = 0;
                flags = 0;
                if (!ext4_es_lookup_extent(inode, block, &next, &es, NULL))
                        break;
                if (ext4_es_is_unwritten(&es))
                        flags |= FIEMAP_EXTENT_UNWRITTEN;
                if (ext4_es_is_delayed(&es))
                        flags |= (FIEMAP_EXTENT_DELALLOC |
                                  FIEMAP_EXTENT_UNKNOWN);
                if (ext4_es_is_hole(&es))
                        flags |= EXT4_FIEMAP_EXTENT_HOLE;
                if (next == 0)
                        flags |= FIEMAP_EXTENT_LAST;
                if (flags & (FIEMAP_EXTENT_DELALLOC|
                             EXT4_FIEMAP_EXTENT_HOLE))
                        es.es_pblk = 0;
                else
                        es.es_pblk = ext4_es_pblock(&es);
                err = fiemap_fill_next_extent(fieinfo,
                                (__u64)es.es_lblk << blksize_bits,
                                (__u64)es.es_pblk << blksize_bits,
                                (__u64)es.es_len << blksize_bits,
                                flags);
                if (next == 0)
                        break;
                block = next;
                if (err < 0)
                        return err;
                if (err == 1)
                        return 0;
        }
        return 0;
}


/*
 * ext4_ext_find_hole - find hole around given block according to the given path
 * @inode:        inode we lookup in
 * @path:        path in extent tree to @lblk
 * @lblk:        pointer to logical block around which we want to determine hole
 *
 * Determine hole length (and start if easily possible) around given logical
 * block. We don't try too hard to find the beginning of the hole but @path
 * actually points to extent before @lblk, we provide it.
 *
 * The function returns the length of a hole starting at @lblk. We update @lblk
 * to the beginning of the hole if we managed to find it.
 */
static ext4_lblk_t ext4_ext_find_hole(struct inode *inode,
                                      struct ext4_ext_path *path,
                                      ext4_lblk_t *lblk)
{
        int depth = ext_depth(inode);
        struct ext4_extent *ex;
        ext4_lblk_t len;

        ex = path[depth].p_ext;
        if (ex == NULL) {
                /* there is no extent yet, so gap is [0;-] */
                *lblk = 0;
                len = EXT_MAX_BLOCKS;
        } else if (*lblk < le32_to_cpu(ex->ee_block)) {
                len = le32_to_cpu(ex->ee_block) - *lblk;
        } else if (*lblk >= le32_to_cpu(ex->ee_block)
                        + ext4_ext_get_actual_len(ex)) {
                ext4_lblk_t next;

                *lblk = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex);
                next = ext4_ext_next_allocated_block(path);
                BUG_ON(next == *lblk);
                len = next - *lblk;
        } else {
                BUG();
        }
        return len;
}

/*
 * ext4_ext_rm_idx:
 * removes index from the index block.
 */
static int ext4_ext_rm_idx(handle_t *handle, struct inode *inode,
                        struct ext4_ext_path *path, int depth)
{
        int err;
        ext4_fsblk_t leaf;
        int k = depth - 1;

        /* free index block */
        leaf = ext4_idx_pblock(path[k].p_idx);
        if (unlikely(path[k].p_hdr->eh_entries == 0)) {
                EXT4_ERROR_INODE(inode, "path[%d].p_hdr->eh_entries == 0", k);
                return -EFSCORRUPTED;
        }
        err = ext4_ext_get_access(handle, inode, path + k);
        if (err)
                return err;

        if (path[k].p_idx != EXT_LAST_INDEX(path[k].p_hdr)) {
                int len = EXT_LAST_INDEX(path[k].p_hdr) - path[k].p_idx;
                len *= sizeof(struct ext4_extent_idx);
                memmove(path[k].p_idx, path[k].p_idx + 1, len);
        }

        le16_add_cpu(&path[k].p_hdr->eh_entries, -1);
        err = ext4_ext_dirty(handle, inode, path + k);
        if (err)
                return err;
        ext_debug(inode, "index is empty, remove it, free block %llu\n", leaf);
        trace_ext4_ext_rm_idx(inode, leaf);

        ext4_free_blocks(handle, inode, NULL, leaf, 1,
                         EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);

        while (--k >= 0) {
                if (path[k + 1].p_idx != EXT_FIRST_INDEX(path[k + 1].p_hdr))
                        break;
                err = ext4_ext_get_access(handle, inode, path + k);
                if (err)
                        goto clean;
                path[k].p_idx->ei_block = path[k + 1].p_idx->ei_block;
                err = ext4_ext_dirty(handle, inode, path + k);
                if (err)
                        goto clean;
        }
        return 0;

clean:
        /*
         * The path[k].p_bh is either unmodified or with no verified bit
         * set (see ext4_ext_get_access()). So just clear the verified bit
         * of the successfully modified extents buffers, which will force
         * these extents to be checked to avoid using inconsistent data.
         */
        while (++k < depth)
                clear_buffer_verified(path[k].p_bh);

        return err;
}

/*
 * ext4_ext_calc_credits_for_single_extent:
 * This routine returns max. credits that needed to insert an extent
 * to the extent tree.
 * When pass the actual path, the caller should calculate credits
 * under i_data_sem.
 */
int ext4_ext_calc_credits_for_single_extent(struct inode *inode, int nrblocks,
                                                struct ext4_ext_path *path)
{
        if (path) {
                int depth = ext_depth(inode);
                int ret = 0;

                /* probably there is space in leaf? */
                if (le16_to_cpu(path[depth].p_hdr->eh_entries)
                                < le16_to_cpu(path[depth].p_hdr->eh_max)) {

                        /*
                         *  There are some space in the leaf tree, no
                         *  need to account for leaf block credit
                         *
                         *  bitmaps and block group descriptor blocks
                         *  and other metadata blocks still need to be
                         *  accounted.
                         */
                        /* 1 bitmap, 1 block group descriptor */
                        ret = 2 + EXT4_META_TRANS_BLOCKS(inode->i_sb);
                        return ret;
                }
        }

        return ext4_chunk_trans_blocks(inode, nrblocks);
}

/*
 * How many index/leaf blocks need to change/allocate to add @extents extents?
 *
 * If we add a single extent, then in the worse case, each tree level
 * index/leaf need to be changed in case of the tree split.
 *
 * If more extents are inserted, they could cause the whole tree split more
 * than once, but this is really rare.
 */
int ext4_ext_index_trans_blocks(struct inode *inode, int extents)
{
        int index;

        /* If we are converting the inline data, only one is needed here. */
        if (ext4_has_inline_data(inode))
                return 1;

        /*
         * Extent tree can change between the time we estimate credits and
         * the time we actually modify the tree. Assume the worst case.
         */
        if (extents <= 1)
                index = (EXT4_MAX_EXTENT_DEPTH * 2) + extents;
        else
                index = (EXT4_MAX_EXTENT_DEPTH * 3) +
                        DIV_ROUND_UP(extents, ext4_ext_space_block(inode, 0));

        return index;
}

static inline int get_default_free_blocks_flags(struct inode *inode)
{
        if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode) ||
            ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE))
                return EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET;
        else if (ext4_should_journal_data(inode))
                return EXT4_FREE_BLOCKS_FORGET;
        return 0;
}

/*
 * ext4_rereserve_cluster - increment the reserved cluster count when
 *                          freeing a cluster with a pending reservation
 *
 * @inode - file containing the cluster
 * @lblk - logical block in cluster to be reserved
 *
 * Increments the reserved cluster count and adjusts quota in a bigalloc
 * file system when freeing a partial cluster containing at least one
 * delayed and unwritten block.  A partial cluster meeting that
 * requirement will have a pending reservation.  If so, the
 * RERESERVE_CLUSTER flag is used when calling ext4_free_blocks() to
 * defer reserved and allocated space accounting to a subsequent call
 * to this function.
 */
static void ext4_rereserve_cluster(struct inode *inode, ext4_lblk_t lblk)
{
        struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
        struct ext4_inode_info *ei = EXT4_I(inode);

        dquot_reclaim_block(inode, EXT4_C2B(sbi, 1));

        spin_lock(&ei->i_block_reservation_lock);
        ei->i_reserved_data_blocks++;
        percpu_counter_add(&sbi->s_dirtyclusters_counter, 1);
        spin_unlock(&ei->i_block_reservation_lock);

        percpu_counter_add(&sbi->s_freeclusters_counter, 1);
        ext4_remove_pending(inode, lblk);
}

static int ext4_remove_blocks(handle_t *handle, struct inode *inode,
                              struct ext4_extent *ex,
                              struct partial_cluster *partial,
                              ext4_lblk_t from, ext4_lblk_t to)
{
        struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
        unsigned short ee_len = ext4_ext_get_actual_len(ex);
        ext4_fsblk_t last_pblk, pblk;
        ext4_lblk_t num;
        int flags;

        /* only extent tail removal is allowed */
        if (from < le32_to_cpu(ex->ee_block) ||
            to != le32_to_cpu(ex->ee_block) + ee_len - 1) {
                ext4_error(sbi->s_sb,
                           "strange request: removal(2) %u-%u from %u:%u",
                           from, to, le32_to_cpu(ex->ee_block), ee_len);
                return 0;
        }

#ifdef EXTENTS_STATS
        spin_lock(&sbi->s_ext_stats_lock);
        sbi->s_ext_blocks += ee_len;
        sbi->s_ext_extents++;
        if (ee_len < sbi->s_ext_min)
                sbi->s_ext_min = ee_len;
        if (ee_len > sbi->s_ext_max)
                sbi->s_ext_max = ee_len;
        if (ext_depth(inode) > sbi->s_depth_max)
                sbi->s_depth_max = ext_depth(inode);
        spin_unlock(&sbi->s_ext_stats_lock);
#endif

        trace_ext4_remove_blocks(inode, ex, from, to, partial);

        /*
         * if we have a partial cluster, and it's different from the
         * cluster of the last block in the extent, we free it
         */
        last_pblk = ext4_ext_pblock(ex) + ee_len - 1;

        if (partial->state != initial &&
            partial->pclu != EXT4_B2C(sbi, last_pblk)) {
                if (partial->state == tofree) {
                        flags = get_default_free_blocks_flags(inode);
                        if (ext4_is_pending(inode, partial->lblk))
                                flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
                        ext4_free_blocks(handle, inode, NULL,
                                         EXT4_C2B(sbi, partial->pclu),
                                         sbi->s_cluster_ratio, flags);
                        if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
                                ext4_rereserve_cluster(inode, partial->lblk);
                }
                partial->state = initial;
        }

        num = le32_to_cpu(ex->ee_block) + ee_len - from;
        pblk = ext4_ext_pblock(ex) + ee_len - num;

        /*
         * We free the partial cluster at the end of the extent (if any),
         * unless the cluster is used by another extent (partial_cluster
         * state is nofree).  If a partial cluster exists here, it must be
         * shared with the last block in the extent.
         */
        flags = get_default_free_blocks_flags(inode);

        /* partial, left end cluster aligned, right end unaligned */
        if ((EXT4_LBLK_COFF(sbi, to) != sbi->s_cluster_ratio - 1) &&
            (EXT4_LBLK_CMASK(sbi, to) >= from) &&
            (partial->state != nofree)) {
                if (ext4_is_pending(inode, to))
                        flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
                ext4_free_blocks(handle, inode, NULL,
                                 EXT4_PBLK_CMASK(sbi, last_pblk),
                                 sbi->s_cluster_ratio, flags);
                if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
                        ext4_rereserve_cluster(inode, to);
                partial->state = initial;
                flags = get_default_free_blocks_flags(inode);
        }

        flags |= EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER;

        /*
         * For bigalloc file systems, we never free a partial cluster
         * at the beginning of the extent.  Instead, we check to see if we
         * need to free it on a subsequent call to ext4_remove_blocks,
         * or at the end of ext4_ext_rm_leaf or ext4_ext_remove_space.
         */
        flags |= EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER;
        ext4_free_blocks(handle, inode, NULL, pblk, num, flags);

        /* reset the partial cluster if we've freed past it */
        if (partial->state != initial && partial->pclu != EXT4_B2C(sbi, pblk))
                partial->state = initial;

        /*
         * If we've freed the entire extent but the beginning is not left
         * cluster aligned and is not marked as ineligible for freeing we
         * record the partial cluster at the beginning of the extent.  It
         * wasn't freed by the preceding ext4_free_blocks() call, and we
         * need to look farther to the left to determine if it's to be freed
         * (not shared with another extent). Else, reset the partial
         * cluster - we're either  done freeing or the beginning of the
         * extent is left cluster aligned.
         */
        if (EXT4_LBLK_COFF(sbi, from) && num == ee_len) {
                if (partial->state == initial) {
                        partial->pclu = EXT4_B2C(sbi, pblk);
                        partial->lblk = from;
                        partial->state = tofree;
                }
        } else {
                partial->state = initial;
        }

        return 0;
}

/*
 * ext4_ext_rm_leaf() Removes the extents associated with the
 * blocks appearing between "start" and "end".  Both "start"
 * and "end" must appear in the same extent or EIO is returned.
 *
 * @handle: The journal handle
 * @inode:  The files inode
 * @path:   The path to the leaf
 * @partial_cluster: The cluster which we'll have to free if all extents
 *                   has been released from it.  However, if this value is
 *                   negative, it's a cluster just to the right of the
 *                   punched region and it must not be freed.
 * @start:  The first block to remove
 * @end:   The last block to remove
 */
static int
ext4_ext_rm_leaf(handle_t *handle, struct inode *inode,
                 struct ext4_ext_path *path,
                 struct partial_cluster *partial,
                 ext4_lblk_t start, ext4_lblk_t end)
{
        struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
        int err = 0, correct_index = 0;
        int depth = ext_depth(inode), credits, revoke_credits;
        struct ext4_extent_header *eh;
        ext4_lblk_t a, b;
        unsigned num;
        ext4_lblk_t ex_ee_block;
        unsigned short ex_ee_len;
        unsigned unwritten = 0;
        struct ext4_extent *ex;
        ext4_fsblk_t pblk;

        /* the header must be checked already in ext4_ext_remove_space() */
        ext_debug(inode, "truncate since %u in leaf to %u\n", start, end);
        if (!path[depth].p_hdr)
                path[depth].p_hdr = ext_block_hdr(path[depth].p_bh);
        eh = path[depth].p_hdr;
        if (unlikely(path[depth].p_hdr == NULL)) {
                EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
                return -EFSCORRUPTED;
        }
        /* find where to start removing */
        ex = path[depth].p_ext;
        if (!ex)
                ex = EXT_LAST_EXTENT(eh);

        ex_ee_block = le32_to_cpu(ex->ee_block);
        ex_ee_len = ext4_ext_get_actual_len(ex);

        trace_ext4_ext_rm_leaf(inode, start, ex, partial);

        while (ex >= EXT_FIRST_EXTENT(eh) &&
                        ex_ee_block + ex_ee_len > start) {

                if (ext4_ext_is_unwritten(ex))
                        unwritten = 1;
                else
                        unwritten = 0;

                ext_debug(inode, "remove ext %u:[%d]%d\n", ex_ee_block,
                          unwritten, ex_ee_len);
                path[depth].p_ext = ex;

                a = max(ex_ee_block, start);
                b = min(ex_ee_block + ex_ee_len - 1, end);

                ext_debug(inode, "  border %u:%u\n", a, b);

                /* If this extent is beyond the end of the hole, skip it */
                if (end < ex_ee_block) {
                        /*
                         * We're going to skip this extent and move to another,
                         * so note that its first cluster is in use to avoid
                         * freeing it when removing blocks.  Eventually, the
                         * right edge of the truncated/punched region will
                         * be just to the left.
                         */
                        if (sbi->s_cluster_ratio > 1) {
                                pblk = ext4_ext_pblock(ex);
                                partial->pclu = EXT4_B2C(sbi, pblk);
                                partial->state = nofree;
                        }
                        ex--;
                        ex_ee_block = le32_to_cpu(ex->ee_block);
                        ex_ee_len = ext4_ext_get_actual_len(ex);
                        continue;
                } else if (b != ex_ee_block + ex_ee_len - 1) {
                        EXT4_ERROR_INODE(inode,
                                         "can not handle truncate %u:%u "
                                         "on extent %u:%u",
                                         start, end, ex_ee_block,
                                         ex_ee_block + ex_ee_len - 1);
                        err = -EFSCORRUPTED;
                        goto out;
                } else if (a != ex_ee_block) {
                        /* remove tail of the extent */
                        num = a - ex_ee_block;
                } else {
                        /* remove whole extent: excellent! */
                        num = 0;
                }
                /*
                 * 3 for leaf, sb, and inode plus 2 (bmap and group
                 * descriptor) for each block group; assume two block
                 * groups plus ex_ee_len/blocks_per_block_group for
                 * the worst case
                 */
                credits = 7 + 2*(ex_ee_len/EXT4_BLOCKS_PER_GROUP(inode->i_sb));
                if (ex == EXT_FIRST_EXTENT(eh)) {
                        correct_index = 1;
                        credits += (ext_depth(inode)) + 1;
                }
                credits += EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb);
                /*
                 * We may end up freeing some index blocks and data from the
                 * punched range. Note that partial clusters are accounted for
                 * by ext4_free_data_revoke_credits().
                 */
                revoke_credits =
                        ext4_free_metadata_revoke_credits(inode->i_sb,
                                                          ext_depth(inode)) +
                        ext4_free_data_revoke_credits(inode, b - a + 1);

                err = ext4_datasem_ensure_credits(handle, inode, credits,
                                                  credits, revoke_credits);
                if (err) {
                        if (err > 0)
                                err = -EAGAIN;
                        goto out;
                }

                err = ext4_ext_get_access(handle, inode, path + depth);
                if (err)
                        goto out;

                err = ext4_remove_blocks(handle, inode, ex, partial, a, b);
                if (err)
                        goto out;

                if (num == 0)
                        /* this extent is removed; mark slot entirely unused */
                        ext4_ext_store_pblock(ex, 0);

                ex->ee_len = cpu_to_le16(num);
                /*
                 * Do not mark unwritten if all the blocks in the
                 * extent have been removed.
                 */
                if (unwritten && num)
                        ext4_ext_mark_unwritten(ex);
                /*
                 * If the extent was completely released,
                 * we need to remove it from the leaf
                 */
                if (num == 0) {
                        if (end != EXT_MAX_BLOCKS - 1) {
                                /*
                                 * For hole punching, we need to scoot all the
                                 * extents up when an extent is removed so that
                                 * we dont have blank extents in the middle
                                 */
                                memmove(ex, ex+1, (EXT_LAST_EXTENT(eh) - ex) *
                                        sizeof(struct ext4_extent));

                                /* Now get rid of the one at the end */
                                memset(EXT_LAST_EXTENT(eh), 0,
                                        sizeof(struct ext4_extent));
                        }
                        le16_add_cpu(&eh->eh_entries, -1);
                }

                err = ext4_ext_dirty(handle, inode, path + depth);
                if (err)
                        goto out;

                ext_debug(inode, "new extent: %u:%u:%llu\n", ex_ee_block, num,
                                ext4_ext_pblock(ex));
                ex--;
                ex_ee_block = le32_to_cpu(ex->ee_block);
                ex_ee_len = ext4_ext_get_actual_len(ex);
        }

        if (correct_index && eh->eh_entries)
                err = ext4_ext_correct_indexes(handle, inode, path);

        /*
         * If there's a partial cluster and at least one extent remains in
         * the leaf, free the partial cluster if it isn't shared with the
         * current extent.  If it is shared with the current extent
         * we reset the partial cluster because we've reached the start of the
         * truncated/punched region and we're done removing blocks.
         */
        if (partial->state == tofree && ex >= EXT_FIRST_EXTENT(eh)) {
                pblk = ext4_ext_pblock(ex) + ex_ee_len - 1;
                if (partial->pclu != EXT4_B2C(sbi, pblk)) {
                        int flags = get_default_free_blocks_flags(inode);

                        if (ext4_is_pending(inode, partial->lblk))
                                flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
                        ext4_free_blocks(handle, inode, NULL,
                                         EXT4_C2B(sbi, partial->pclu),
                                         sbi->s_cluster_ratio, flags);
                        if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
                                ext4_rereserve_cluster(inode, partial->lblk);
                }
                partial->state = initial;
        }

        /* if this leaf is free, then we should
         * remove it from index block above */
        if (err == 0 && eh->eh_entries == 0 && path[depth].p_bh != NULL)
                err = ext4_ext_rm_idx(handle, inode, path, depth);

out:
        return err;
}

/*
 * ext4_ext_more_to_rm:
 * returns 1 if current index has to be freed (even partial)
 */
static int
ext4_ext_more_to_rm(struct ext4_ext_path *path)
{
        BUG_ON(path->p_idx == NULL);

        if (path->p_idx < EXT_FIRST_INDEX(path->p_hdr))
                return 0;

        /*
         * if truncate on deeper level happened, it wasn't partial,
         * so we have to consider current index for truncation
         */
        if (le16_to_cpu(path->p_hdr->eh_entries) == path->p_block)
                return 0;
        return 1;
}

int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start,
                          ext4_lblk_t end)
{
        struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
        int depth = ext_depth(inode);
        struct ext4_ext_path *path = NULL;
        struct partial_cluster partial;
        handle_t *handle;
        int i = 0, err = 0;
        int flags = EXT4_EX_NOCACHE | EXT4_EX_NOFAIL;

        partial.pclu = 0;
        partial.lblk = 0;
        partial.state = initial;

        ext_debug(inode, "truncate since %u to %u\n", start, end);

        /* probably first extent we're gonna free will be last in block */
        handle = ext4_journal_start_with_revoke(inode, EXT4_HT_TRUNCATE,
                        depth + 1,
                        ext4_free_metadata_revoke_credits(inode->i_sb, depth));
        if (IS_ERR(handle))
                return PTR_ERR(handle);

again:
        trace_ext4_ext_remove_space(inode, start, end, depth);

        /*
         * Check if we are removing extents inside the extent tree. If that
         * is the case, we are going to punch a hole inside the extent tree
         * so we have to check whether we need to split the extent covering
         * the last block to remove so we can easily remove the part of it
         * in ext4_ext_rm_leaf().
         */
        if (end < EXT_MAX_BLOCKS - 1) {
                struct ext4_extent *ex;
                ext4_lblk_t ee_block, ex_end, lblk;
                ext4_fsblk_t pblk;

                /* find extent for or closest extent to this block */
                path = ext4_find_extent(inode, end, NULL, flags);
                if (IS_ERR(path)) {
                        ext4_journal_stop(handle);
                        return PTR_ERR(path);
                }
                depth = ext_depth(inode);
                /* Leaf not may not exist only if inode has no blocks at all */
                ex = path[depth].p_ext;
                if (!ex) {
                        if (depth) {
                                EXT4_ERROR_INODE(inode,
                                                 "path[%d].p_hdr == NULL",
                                                 depth);
                                err = -EFSCORRUPTED;
                        }
                        goto out;
                }

                ee_block = le32_to_cpu(ex->ee_block);
                ex_end = ee_block + ext4_ext_get_actual_len(ex) - 1;

                /*
                 * See if the last block is inside the extent, if so split
                 * the extent at 'end' block so we can easily remove the
                 * tail of the first part of the split extent in
                 * ext4_ext_rm_leaf().
                 */
                if (end >= ee_block && end < ex_end) {

                        /*
                         * If we're going to split the extent, note that
                         * the cluster containing the block after 'end' is
                         * in use to avoid freeing it when removing blocks.
                         */
                        if (sbi->s_cluster_ratio > 1) {
                                pblk = ext4_ext_pblock(ex) + end - ee_block + 1;
                                partial.pclu = EXT4_B2C(sbi, pblk);
                                partial.state = nofree;
                        }

                        /*
                         * Split the extent in two so that 'end' is the last
                         * block in the first new extent. Also we should not
                         * fail removing space due to ENOSPC so try to use
                         * reserved block if that happens.
                         */
                        path = ext4_force_split_extent_at(handle, inode, path,
                                                          end + 1, 1);
                        if (IS_ERR(path)) {
                                err = PTR_ERR(path);
                                goto out;
                        }
                } else if (sbi->s_cluster_ratio > 1 && end >= ex_end &&
                           partial.state == initial) {
                        /*
                         * If we're punching, there's an extent to the right.
                         * If the partial cluster hasn't been set, set it to
                         * that extent's first cluster and its state to nofree
                         * so it won't be freed should it contain blocks to be
                         * removed. If it's already set (tofree/nofree), we're
                         * retrying and keep the original partial cluster info
                         * so a cluster marked tofree as a result of earlier
                         * extent removal is not lost.
                         */
                        lblk = ex_end + 1;
                        err = ext4_ext_search_right(inode, path, &lblk, &pblk,
                                                    NULL, flags);
                        if (err < 0)
                                goto out;
                        if (pblk) {
                                partial.pclu = EXT4_B2C(sbi, pblk);
                                partial.state = nofree;
                        }
                }
        }
        /*
         * We start scanning from right side, freeing all the blocks
         * after i_size and walking into the tree depth-wise.
         */
        depth = ext_depth(inode);
        if (path) {
                int k = i = depth;
                while (--k > 0)
                        path[k].p_block =
                                le16_to_cpu(path[k].p_hdr->eh_entries)+1;
        } else {
                path = kzalloc_objs(struct ext4_ext_path, depth + 1,
                                    GFP_NOFS | __GFP_NOFAIL);
                path[0].p_maxdepth = path[0].p_depth = depth;
                path[0].p_hdr = ext_inode_hdr(inode);
                i = 0;

                if (ext4_ext_check(inode, path[0].p_hdr, depth, 0)) {
                        err = -EFSCORRUPTED;
                        goto out;
                }
        }
        err = 0;

        while (i >= 0 && err == 0) {
                if (i == depth) {
                        /* this is leaf block */
                        err = ext4_ext_rm_leaf(handle, inode, path,
                                               &partial, start, end);
                        /* root level has p_bh == NULL, brelse() eats this */
                        ext4_ext_path_brelse(path + i);
                        i--;
                        continue;
                }

                /* this is index block */
                if (!path[i].p_hdr) {
                        ext_debug(inode, "initialize header\n");
                        path[i].p_hdr = ext_block_hdr(path[i].p_bh);
                }

                if (!path[i].p_idx) {
                        /* this level hasn't been touched yet */
                        path[i].p_idx = EXT_LAST_INDEX(path[i].p_hdr);
                        path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries)+1;
                        ext_debug(inode, "init index ptr: hdr 0x%p, num %d\n",
                                  path[i].p_hdr,
                                  le16_to_cpu(path[i].p_hdr->eh_entries));
                } else {
                        /* we were already here, see at next index */
                        path[i].p_idx--;
                }

                ext_debug(inode, "level %d - index, first 0x%p, cur 0x%p\n",
                                i, EXT_FIRST_INDEX(path[i].p_hdr),
                                path[i].p_idx);
                if (ext4_ext_more_to_rm(path + i)) {
                        struct buffer_head *bh;
                        /* go to the next level */
                        ext_debug(inode, "move to level %d (block %llu)\n",
                                  i + 1, ext4_idx_pblock(path[i].p_idx));
                        memset(path + i + 1, 0, sizeof(*path));
                        bh = read_extent_tree_block(inode, path[i].p_idx,
                                                    depth - i - 1, flags);
                        if (IS_ERR(bh)) {
                                /* should we reset i_size? */
                                err = PTR_ERR(bh);
                                break;
                        }
                        /* Yield here to deal with large extent trees.
                         * Should be a no-op if we did IO above. */
                        cond_resched();
                        if (WARN_ON(i + 1 > depth)) {
                                err = -EFSCORRUPTED;
                                break;
                        }
                        path[i + 1].p_bh = bh;

                        /* save actual number of indexes since this
                         * number is changed at the next iteration */
                        path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries);
                        i++;
                } else {
                        /* we finished processing this index, go up */
                        if (path[i].p_hdr->eh_entries == 0 && i > 0) {
                                /* index is empty, remove it;
                                 * handle must be already prepared by the
                                 * truncatei_leaf() */
                                err = ext4_ext_rm_idx(handle, inode, path, i);
                        }
                        /* root level has p_bh == NULL, brelse() eats this */
                        ext4_ext_path_brelse(path + i);
                        i--;
                        ext_debug(inode, "return to level %d\n", i);
                }
        }

        trace_ext4_ext_remove_space_done(inode, start, end, depth, &partial,
                                         path->p_hdr->eh_entries);

        /*
         * if there's a partial cluster and we have removed the first extent
         * in the file, then we also free the partial cluster, if any
         */
        if (partial.state == tofree && err == 0) {
                int flags = get_default_free_blocks_flags(inode);

                if (ext4_is_pending(inode, partial.lblk))
                        flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
                ext4_free_blocks(handle, inode, NULL,
                                 EXT4_C2B(sbi, partial.pclu),
                                 sbi->s_cluster_ratio, flags);
                if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
                        ext4_rereserve_cluster(inode, partial.lblk);
                partial.state = initial;
        }

        /* TODO: flexible tree reduction should be here */
        if (path->p_hdr->eh_entries == 0) {
                /*
                 * truncate to zero freed all the tree,
                 * so we need to correct eh_depth
                 */
                err = ext4_ext_get_access(handle, inode, path);
                if (err == 0) {
                        ext_inode_hdr(inode)->eh_depth = 0;
                        ext_inode_hdr(inode)->eh_max =
                                cpu_to_le16(ext4_ext_space_root(inode, 0));
                        err = ext4_ext_dirty(handle, inode, path);
                }
        }
out:
        ext4_free_ext_path(path);
        path = NULL;
        if (err == -EAGAIN)
                goto again;
        ext4_journal_stop(handle);

        return err;
}

/*
 * called at mount time
 */
void ext4_ext_init(struct super_block *sb)
{
        /*
         * possible initialization would be here
         */

        if (ext4_has_feature_extents(sb)) {
#if defined(AGGRESSIVE_TEST) || defined(CHECK_BINSEARCH) || defined(EXTENTS_STATS)
                printk(KERN_INFO "EXT4-fs: file extents enabled"
#ifdef AGGRESSIVE_TEST
                       ", aggressive tests"
#endif
#ifdef CHECK_BINSEARCH
                       ", check binsearch"
#endif
#ifdef EXTENTS_STATS
                       ", stats"
#endif
                       "\n");
#endif
#ifdef EXTENTS_STATS
                spin_lock_init(&EXT4_SB(sb)->s_ext_stats_lock);
                EXT4_SB(sb)->s_ext_min = 1 << 30;
                EXT4_SB(sb)->s_ext_max = 0;
#endif
        }
}

/*
 * called at umount time
 */
void ext4_ext_release(struct super_block *sb)
{
        if (!ext4_has_feature_extents(sb))
                return;

#ifdef EXTENTS_STATS
        if (EXT4_SB(sb)->s_ext_blocks && EXT4_SB(sb)->s_ext_extents) {
                struct ext4_sb_info *sbi = EXT4_SB(sb);
                printk(KERN_ERR "EXT4-fs: %lu blocks in %lu extents (%lu ave)\n",
                        sbi->s_ext_blocks, sbi->s_ext_extents,
                        sbi->s_ext_blocks / sbi->s_ext_extents);
                printk(KERN_ERR "EXT4-fs: extents: %lu min, %lu max, max depth %lu\n",
                        sbi->s_ext_min, sbi->s_ext_max, sbi->s_depth_max);
        }
#endif
}

static void ext4_zeroout_es(struct inode *inode, struct ext4_extent *ex)
{
        ext4_lblk_t  ee_block;
        ext4_fsblk_t ee_pblock;
        unsigned int ee_len;

        ee_block = le32_to_cpu(ex->ee_block);
        ee_len = ext4_ext_get_actual_len(ex);
        ee_pblock = ext4_ext_pblock(ex);

        if (ee_len == 0)
                return;

        ext4_es_insert_extent(inode, ee_block, ee_len, ee_pblock,
                              EXTENT_STATUS_WRITTEN, false);
}

/* FIXME!! we need to try to merge to left or right after zero-out  */
int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex)
{
        ext4_fsblk_t ee_pblock;
        unsigned int ee_len;

        KUNIT_STATIC_STUB_REDIRECT(ext4_ext_zeroout, inode, ex);

        ee_len    = ext4_ext_get_actual_len(ex);
        ee_pblock = ext4_ext_pblock(ex);
        return ext4_issue_zeroout(inode, le32_to_cpu(ex->ee_block), ee_pblock,
                                  ee_len);
}

/*
 * ext4_split_extent_at() splits an extent at given block.
 *
 * @handle: the journal handle
 * @inode: the file inode
 * @path: the path to the extent
 * @split: the logical block where the extent is splitted.
 * @flags: flags used to insert new extent to extent tree.
 *
 *
 * Splits extent [a, b] into two extents [a, @split) and [@split, b], states
 * of which are same as the original extent. No conversion is performed.
 *
 * Return an extent path pointer on success, or an error pointer on failure. On
 * failure, the extent is restored to original state.
 */
static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle,
                                                  struct inode *inode,
                                                  struct ext4_ext_path *path,
                                                  ext4_lblk_t split,
                                                  int flags)
{
        ext4_fsblk_t newblock;
        ext4_lblk_t ee_block;
        struct ext4_extent *ex, newex, orig_ex;
        struct ext4_extent *ex2 = NULL;
        unsigned int ee_len, depth;
        int err = 0, insert_err = 0, is_unwrit = 0;

        /* Do not cache extents that are in the process of being modified. */
        flags |= EXT4_EX_NOCACHE;

        ext_debug(inode, "logical block %llu\n", (unsigned long long)split);

        ext4_ext_show_leaf(inode, path);

        depth = ext_depth(inode);
        ex = path[depth].p_ext;
        ee_block = le32_to_cpu(ex->ee_block);
        ee_len = ext4_ext_get_actual_len(ex);
        newblock = split - ee_block + ext4_ext_pblock(ex);
        is_unwrit = ext4_ext_is_unwritten(ex);

        BUG_ON(split < ee_block || split >= (ee_block + ee_len));

        /*
         * No split needed
         */
        if (split == ee_block)
                goto out;

        err = ext4_ext_get_access(handle, inode, path + depth);
        if (err)
                goto out;

        /* case a */
        memcpy(&orig_ex, ex, sizeof(orig_ex));
        ex->ee_len = cpu_to_le16(split - ee_block);
        if (is_unwrit)
                ext4_ext_mark_unwritten(ex);

        /*
         * path may lead to new leaf, not to original leaf any more
         * after ext4_ext_insert_extent() returns,
         */
        err = ext4_ext_dirty(handle, inode, path + depth);
        if (err)
                goto fix_extent_len;

        ex2 = &newex;
        ex2->ee_block = cpu_to_le32(split);
        ex2->ee_len   = cpu_to_le16(ee_len - (split - ee_block));
        ext4_ext_store_pblock(ex2, newblock);
        if (is_unwrit)
                ext4_ext_mark_unwritten(ex2);

        path = ext4_ext_insert_extent(handle, inode, path, &newex, flags);
        if (!IS_ERR(path))
                return path;

        insert_err = PTR_ERR(path);
        err = 0;
        if (insert_err != -ENOSPC && insert_err != -EDQUOT &&
            insert_err != -ENOMEM)
                goto out_path;

        /*
         * Get a new path to try to zeroout or fix the extent length.
         * Using EXT4_EX_NOFAIL guarantees that ext4_find_extent()
         * will not return -ENOMEM, otherwise -ENOMEM will cause a
         * retry in do_writepages(), and a WARN_ON may be triggered
         * in ext4_da_update_reserve_space() due to an incorrect
         * ee_len causing the i_reserved_data_blocks exception.
         */
        path = ext4_find_extent(inode, ee_block, NULL, flags | EXT4_EX_NOFAIL);
        if (IS_ERR(path)) {
                EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %ld",
                                 split, PTR_ERR(path));
                goto out_path;
        }

        depth = ext_depth(inode);
        ex = path[depth].p_ext;
        if (!ex) {
                EXT4_ERROR_INODE(inode,
                                 "bad extent address lblock: %lu, depth: %d pblock %llu",
                                 (unsigned long)ee_block, depth, path[depth].p_block);
                err = -EFSCORRUPTED;
                goto out;
        }

        err = ext4_ext_get_access(handle, inode, path + depth);
        if (err)
                goto out;

fix_extent_len:
        ex->ee_len = orig_ex.ee_len;
        err = ext4_ext_dirty(handle, inode, path + path->p_depth);
out:
        if (err || insert_err) {
                ext4_free_ext_path(path);
                path = err ? ERR_PTR(err) : ERR_PTR(insert_err);
        }
out_path:
        if (IS_ERR(path))
                /* Remove all remaining potentially stale extents. */
                ext4_es_remove_extent(inode, ee_block, ee_len);
        ext4_ext_show_leaf(inode, path);
        return path;
}

static int ext4_split_extent_zeroout(handle_t *handle, struct inode *inode,
                                     struct ext4_ext_path *path,
                                     struct ext4_map_blocks *map, int flags)
{
        struct ext4_extent *ex;
        unsigned int ee_len, depth;
        ext4_lblk_t ee_block;
        uint64_t lblk, pblk, len;
        int is_unwrit;
        int err = 0;

        depth = ext_depth(inode);
        ex = path[depth].p_ext;
        ee_block = le32_to_cpu(ex->ee_block);
        ee_len = ext4_ext_get_actual_len(ex);
        is_unwrit = ext4_ext_is_unwritten(ex);

        if (flags & EXT4_GET_BLOCKS_CONVERT) {
                /*
                 * EXT4_GET_BLOCKS_CONVERT: Caller wants the range specified by
                 * map to be initialized. Zeroout everything except the map
                 * range.
                 */

                loff_t map_end = (loff_t) map->m_lblk + map->m_len;
                loff_t ex_end = (loff_t) ee_block + ee_len;

                if (!is_unwrit)
                        /* Shouldn't happen. Just exit */
                        return -EINVAL;

                /* zeroout left */
                if (map->m_lblk > ee_block) {
                        lblk = ee_block;
                        len = map->m_lblk - ee_block;
                        pblk = ext4_ext_pblock(ex);
                        err = ext4_issue_zeroout(inode, lblk, pblk, len);
                        if (err)
                                /* ZEROOUT failed, just return original error */
                                return err;
                }

                /* zeroout right */
                if (map_end < ex_end) {
                        lblk = map_end;
                        len = ex_end - map_end;
                        pblk = ext4_ext_pblock(ex) + (map_end - ee_block);
                        err = ext4_issue_zeroout(inode, lblk, pblk, len);
                        if (err)
                                /* ZEROOUT failed, just return original error */
                                return err;
                }
        } else if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) {
                /*
                 * EXT4_GET_BLOCKS_CONVERT_UNWRITTEN: Caller wants the
                 * range specified by map to be marked unwritten.
                 * Zeroout the map range leaving rest as it is.
                 */

                if (is_unwrit)
                        /* Shouldn't happen. Just exit */
                        return -EINVAL;

                lblk = map->m_lblk;
                len = map->m_len;
                pblk = ext4_ext_pblock(ex) + (map->m_lblk - ee_block);
                err = ext4_issue_zeroout(inode, lblk, pblk, len);
                if (err)
                        /* ZEROOUT failed, just return original error */
                        return err;
        } else {
                /*
                 * We no longer perform unwritten to unwritten splits in IO paths.
                 * Hence this should not happen.
                 */
                WARN_ON_ONCE(true);
                return -EINVAL;
        }

        err = ext4_ext_get_access(handle, inode, path + depth);
        if (err)
                return err;

        ext4_ext_mark_initialized(ex);

        err = ext4_ext_dirty(handle, inode, path + depth);
        if (err)
                return err;

        return 0;
}

/*
 * ext4_split_extent() splits an extent and mark extent which is covered
 * by @map as split_flags indicates
 *
 * It may result in splitting the extent into multiple extents (up to three)
 * There are three possibilities:
 *   a> There is no split required
 *   b> Splits in two extents: Split is happening at either end of the extent
 *   c> Splits in three extents: Somone is splitting in middle of the extent
 *
 */
static struct ext4_ext_path *ext4_split_extent(handle_t *handle,
                                               struct inode *inode,
                                               struct ext4_ext_path *path,
                                               struct ext4_map_blocks *map,
                                               int split_flag, int flags,
                                               unsigned int *allocated, bool *did_zeroout)
{
        ext4_lblk_t ee_block, orig_ee_block;
        struct ext4_extent *ex;
        unsigned int ee_len, orig_ee_len, depth;
        int unwritten, orig_unwritten;
        int orig_err = 0;

        depth = ext_depth(inode);
        ex = path[depth].p_ext;
        ee_block = le32_to_cpu(ex->ee_block);
        ee_len = ext4_ext_get_actual_len(ex);
        unwritten = ext4_ext_is_unwritten(ex);

        orig_ee_block = ee_block;
        orig_ee_len = ee_len;
        orig_unwritten = unwritten;

        /* Do not cache extents that are in the process of being modified. */
        flags |= EXT4_EX_NOCACHE;

        if (map->m_lblk + map->m_len < ee_block + ee_len) {
                path = ext4_split_extent_at(handle, inode, path,
                                            map->m_lblk + map->m_len, flags);
                if (IS_ERR(path))
                        goto try_zeroout;

                /*
                 * Update path is required because previous ext4_split_extent_at
                 * may result in split of original leaf or extent zeroout.
                 */
                path = ext4_find_extent(inode, map->m_lblk, path, flags);
                if (IS_ERR(path))
                        goto try_zeroout;

                depth = ext_depth(inode);
                ex = path[depth].p_ext;
                if (!ex) {
                        EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
                                        (unsigned long) map->m_lblk);
                        ext4_free_ext_path(path);
                        return ERR_PTR(-EFSCORRUPTED);
                }

                /* extent would have changed so update original values */
                orig_ee_block = le32_to_cpu(ex->ee_block);
                orig_ee_len = ext4_ext_get_actual_len(ex);
                orig_unwritten = ext4_ext_is_unwritten(ex);
        }

        if (map->m_lblk >= ee_block) {
                path = ext4_split_extent_at(handle, inode, path, map->m_lblk,
                                            flags);
                if (IS_ERR(path))
                        goto try_zeroout;
        }

        goto success;

try_zeroout:
        /*
         * There was an error in splitting the extent. So instead, just zeroout
         * unwritten portions and convert it to initialized as a last resort. If
         * there is any failure here we just return the original error
         */

        orig_err = PTR_ERR(path);
        if (orig_err != -ENOSPC && orig_err != -EDQUOT && orig_err != -ENOMEM)
                goto out_orig_err;

        /* we can't zeroout? just return the original err */
        if (!(split_flag & EXT4_EXT_MAY_ZEROOUT))
                goto out_orig_err;

        if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) {
                int max_zeroout_blks =
                        EXT4_SB(inode->i_sb)->s_extent_max_zeroout_kb >>
                        (inode->i_sb->s_blocksize_bits - 10);

                if (map->m_len > max_zeroout_blks)
                        goto out_orig_err;
        }

        path = ext4_find_extent(inode, map->m_lblk, NULL, flags);
        if (IS_ERR(path))
                goto out_orig_err;

        depth = ext_depth(inode);
        ex = path[depth].p_ext;
        ee_block = le32_to_cpu(ex->ee_block);
        ee_len = ext4_ext_get_actual_len(ex);
        unwritten = ext4_ext_is_unwritten(ex);

        /* extent to zeroout should have been unchanged but its not */
        if (WARN_ON(ee_block != orig_ee_block || ee_len != orig_ee_len ||
                    unwritten != orig_unwritten))
                goto out_free_path;

        if (ext4_split_extent_zeroout(handle, inode, path, map, flags))
                goto out_free_path;

        /* zeroout succeeded */
        if (did_zeroout)
                *did_zeroout = true;

success:
        if (allocated) {
                if (map->m_lblk + map->m_len > ee_block + ee_len)
                        *allocated = ee_len - (map->m_lblk - ee_block);
                else
                        *allocated = map->m_len;
        }
        ext4_ext_show_leaf(inode, path);
        return path;

out_free_path:
        ext4_free_ext_path(path);
out_orig_err:
        return ERR_PTR(orig_err);

}

/*
 * This function is called by ext4_ext_map_blocks() if someone tries to write
 * to an unwritten extent. It may result in splitting the unwritten
 * extent into multiple extents (up to three - one initialized and two
 * unwritten).
 * There are three possibilities:
 *   a> There is no split required: Entire extent should be initialized
 *   b> Splits in two extents: Write is happening at either end of the extent
 *   c> Splits in three extents: Somone is writing in middle of the extent
 *
 * Pre-conditions:
 *  - The extent pointed to by 'path' is unwritten.
 *  - The extent pointed to by 'path' contains a superset
 *    of the logical span [map->m_lblk, map->m_lblk + map->m_len).
 *
 * Post-conditions on success:
 *  - the returned value is the number of blocks beyond map->l_lblk
 *    that are allocated and initialized.
 *    It is guaranteed to be >= map->m_len.
 */
static struct ext4_ext_path *
ext4_ext_convert_to_initialized(handle_t *handle, struct inode *inode,
                        struct ext4_map_blocks *map, struct ext4_ext_path *path,
                        int flags, unsigned int *allocated)
{
        struct ext4_sb_info *sbi;
        struct ext4_extent_header *eh;
        struct ext4_map_blocks split_map;
        struct ext4_extent zero_ex1, zero_ex2;
        struct ext4_extent *ex, *abut_ex;
        ext4_lblk_t ee_block, eof_block;
        unsigned int ee_len, depth, map_len = map->m_len;
        int err = 0;
        unsigned int max_zeroout = 0;

        ext_debug(inode, "logical block %llu, max_blocks %u\n",
                  (unsigned long long)map->m_lblk, map_len);

        sbi = EXT4_SB(inode->i_sb);
        eof_block = (EXT4_I(inode)->i_disksize + inode->i_sb->s_blocksize - 1)
                        >> inode->i_sb->s_blocksize_bits;
        if (eof_block < map->m_lblk + map_len)
                eof_block = map->m_lblk + map_len;

        depth = ext_depth(inode);
        eh = path[depth].p_hdr;
        ex = path[depth].p_ext;
        ee_block = le32_to_cpu(ex->ee_block);
        ee_len = ext4_ext_get_actual_len(ex);
        zero_ex1.ee_len = 0;
        zero_ex2.ee_len = 0;

        trace_ext4_ext_convert_to_initialized_enter(inode, map, ex);

        /* Pre-conditions */
        BUG_ON(!ext4_ext_is_unwritten(ex));
        BUG_ON(!in_range(map->m_lblk, ee_block, ee_len));

        /*
         * Attempt to transfer newly initialized blocks from the currently
         * unwritten extent to its neighbor. This is much cheaper
         * than an insertion followed by a merge as those involve costly
         * memmove() calls. Transferring to the left is the common case in
         * steady state for workloads doing fallocate(FALLOC_FL_KEEP_SIZE)
         * followed by append writes.
         *
         * Limitations of the current logic:
         *  - L1: we do not deal with writes covering the whole extent.
         *    This would require removing the extent if the transfer
         *    is possible.
         *  - L2: we only attempt to merge with an extent stored in the
         *    same extent tree node.
         */
        *allocated = 0;
        if ((map->m_lblk == ee_block) &&
                /* See if we can merge left */
                (map_len < ee_len) &&                /*L1*/
                (ex > EXT_FIRST_EXTENT(eh))) {        /*L2*/
                ext4_lblk_t prev_lblk;
                ext4_fsblk_t prev_pblk, ee_pblk;
                unsigned int prev_len;

                abut_ex = ex - 1;
                prev_lblk = le32_to_cpu(abut_ex->ee_block);
                prev_len = ext4_ext_get_actual_len(abut_ex);
                prev_pblk = ext4_ext_pblock(abut_ex);
                ee_pblk = ext4_ext_pblock(ex);

                /*
                 * A transfer of blocks from 'ex' to 'abut_ex' is allowed
                 * upon those conditions:
                 * - C1: abut_ex is initialized,
                 * - C2: abut_ex is logically abutting ex,
                 * - C3: abut_ex is physically abutting ex,
                 * - C4: abut_ex can receive the additional blocks without
                 *   overflowing the (initialized) length limit.
                 */
                if ((!ext4_ext_is_unwritten(abut_ex)) &&                /*C1*/
                        ((prev_lblk + prev_len) == ee_block) &&                /*C2*/
                        ((prev_pblk + prev_len) == ee_pblk) &&                /*C3*/
                        (prev_len < (EXT_INIT_MAX_LEN - map_len))) {        /*C4*/
                        err = ext4_ext_get_access(handle, inode, path + depth);
                        if (err)
                                goto errout;

                        trace_ext4_ext_convert_to_initialized_fastpath(inode,
                                map, ex, abut_ex);

                        /* Shift the start of ex by 'map_len' blocks */
                        ex->ee_block = cpu_to_le32(ee_block + map_len);
                        ext4_ext_store_pblock(ex, ee_pblk + map_len);
                        ex->ee_len = cpu_to_le16(ee_len - map_len);
                        ext4_ext_mark_unwritten(ex); /* Restore the flag */

                        /* Extend abut_ex by 'map_len' blocks */
                        abut_ex->ee_len = cpu_to_le16(prev_len + map_len);

                        /* Result: number of initialized blocks past m_lblk */
                        *allocated = map_len;
                }
        } else if (((map->m_lblk + map_len) == (ee_block + ee_len)) &&
                   (map_len < ee_len) &&        /*L1*/
                   ex < EXT_LAST_EXTENT(eh)) {        /*L2*/
                /* See if we can merge right */
                ext4_lblk_t next_lblk;
                ext4_fsblk_t next_pblk, ee_pblk;
                unsigned int next_len;

                abut_ex = ex + 1;
                next_lblk = le32_to_cpu(abut_ex->ee_block);
                next_len = ext4_ext_get_actual_len(abut_ex);
                next_pblk = ext4_ext_pblock(abut_ex);
                ee_pblk = ext4_ext_pblock(ex);

                /*
                 * A transfer of blocks from 'ex' to 'abut_ex' is allowed
                 * upon those conditions:
                 * - C1: abut_ex is initialized,
                 * - C2: abut_ex is logically abutting ex,
                 * - C3: abut_ex is physically abutting ex,
                 * - C4: abut_ex can receive the additional blocks without
                 *   overflowing the (initialized) length limit.
                 */
                if ((!ext4_ext_is_unwritten(abut_ex)) &&                /*C1*/
                    ((map->m_lblk + map_len) == next_lblk) &&                /*C2*/
                    ((ee_pblk + ee_len) == next_pblk) &&                /*C3*/
                    (next_len < (EXT_INIT_MAX_LEN - map_len))) {        /*C4*/
                        err = ext4_ext_get_access(handle, inode, path + depth);
                        if (err)
                                goto errout;

                        trace_ext4_ext_convert_to_initialized_fastpath(inode,
                                map, ex, abut_ex);

                        /* Shift the start of abut_ex by 'map_len' blocks */
                        abut_ex->ee_block = cpu_to_le32(next_lblk - map_len);
                        ext4_ext_store_pblock(abut_ex, next_pblk - map_len);
                        ex->ee_len = cpu_to_le16(ee_len - map_len);
                        ext4_ext_mark_unwritten(ex); /* Restore the flag */

                        /* Extend abut_ex by 'map_len' blocks */
                        abut_ex->ee_len = cpu_to_le16(next_len + map_len);

                        /* Result: number of initialized blocks past m_lblk */
                        *allocated = map_len;
                }
        }
        if (*allocated) {
                /* Mark the block containing both extents as dirty */
                err = ext4_ext_dirty(handle, inode, path + depth);

                /* Update path to point to the right extent */
                path[depth].p_ext = abut_ex;
                if (err)
                        goto errout;
                goto out;
        } else
                *allocated = ee_len - (map->m_lblk - ee_block);

        WARN_ON(map->m_lblk < ee_block);
        /*
         * It is safe to convert extent to initialized via explicit
         * zeroout only if extent is fully inside i_size or new_size.
         */
        if (ee_block + ee_len <= eof_block)
                max_zeroout = sbi->s_extent_max_zeroout_kb >>
                        (inode->i_sb->s_blocksize_bits - 10);

        /*
         * five cases:
         * 1. split the extent into three extents.
         * 2. split the extent into two extents, zeroout the head of the first
         *    extent.
         * 3. split the extent into two extents, zeroout the tail of the second
         *    extent.
         * 4. split the extent into two extents with out zeroout.
         * 5. no splitting needed, just possibly zeroout the head and / or the
         *    tail of the extent.
         */
        split_map.m_lblk = map->m_lblk;
        split_map.m_len = map->m_len;

        if (max_zeroout && (*allocated > split_map.m_len)) {
                if (*allocated <= max_zeroout) {
                        /* case 3 or 5 */
                        zero_ex1.ee_block =
                                 cpu_to_le32(split_map.m_lblk +
                                             split_map.m_len);
                        zero_ex1.ee_len =
                                cpu_to_le16(*allocated - split_map.m_len);
                        ext4_ext_store_pblock(&zero_ex1,
                                ext4_ext_pblock(ex) + split_map.m_lblk +
                                split_map.m_len - ee_block);
                        err = ext4_ext_zeroout(inode, &zero_ex1);
                        if (err)
                                goto fallback;
                        split_map.m_len = *allocated;
                }
                if (split_map.m_lblk - ee_block + split_map.m_len <
                                                                max_zeroout) {
                        /* case 2 or 5 */
                        if (split_map.m_lblk != ee_block) {
                                zero_ex2.ee_block = ex->ee_block;
                                zero_ex2.ee_len = cpu_to_le16(split_map.m_lblk -
                                                        ee_block);
                                ext4_ext_store_pblock(&zero_ex2,
                                                      ext4_ext_pblock(ex));
                                err = ext4_ext_zeroout(inode, &zero_ex2);
                                if (err)
                                        goto fallback;
                        }

                        split_map.m_len += split_map.m_lblk - ee_block;
                        split_map.m_lblk = ee_block;
                        *allocated = map->m_len;
                }
        }

fallback:
        path = ext4_split_convert_extents(handle, inode, &split_map, path,
                                          flags | EXT4_GET_BLOCKS_CONVERT, NULL);
        if (IS_ERR(path))
                return path;
out:
        /* If we have gotten a failure, don't zero out status tree */
        ext4_zeroout_es(inode, &zero_ex1);
        ext4_zeroout_es(inode, &zero_ex2);
        return path;

errout:
        ext4_free_ext_path(path);
        return ERR_PTR(err);
}

/*
 * This function is called by ext4_ext_map_blocks() from
 * ext4_get_blocks_dio_write() when DIO to write
 * to an unwritten extent.
 *
 * Writing to an unwritten extent may result in splitting the unwritten
 * extent into multiple initialized/unwritten extents (up to three)
 * There are three possibilities:
 *   a> There is no split required: Entire extent should be unwritten
 *   b> Splits in two extents: Write is happening at either end of the extent
 *   c> Splits in three extents: Somone is writing in middle of the extent
 *
 * This works the same way in the case of initialized -> unwritten conversion.
 *
 * One of more index blocks maybe needed if the extent tree grow after
 * the unwritten extent split. To prevent ENOSPC occur at the IO
 * complete, we need to split the unwritten extent before DIO submit
 * the IO. The unwritten extent called at this time will be split
 * into three unwritten extent(at most). After IO complete, the part
 * being filled will be convert to initialized by the end_io callback function
 * via ext4_convert_unwritten_extents().
 *
 * The size of unwritten extent to be written is passed to the caller via the
 * allocated pointer. Return an extent path pointer on success, or an error
 * pointer on failure.
 */
static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle,
                                        struct inode *inode,
                                        struct ext4_map_blocks *map,
                                        struct ext4_ext_path *path,
                                        int flags, unsigned int *allocated)
{
        ext4_lblk_t eof_block;
        ext4_lblk_t ee_block;
        struct ext4_extent *ex;
        unsigned int ee_len;
        int split_flag = 0, depth, err = 0;
        bool did_zeroout = false;

        ext_debug(inode, "logical block %llu, max_blocks %u\n",
                  (unsigned long long)map->m_lblk, map->m_len);

        eof_block = (EXT4_I(inode)->i_disksize + inode->i_sb->s_blocksize - 1)
                        >> inode->i_sb->s_blocksize_bits;
        if (eof_block < map->m_lblk + map->m_len)
                eof_block = map->m_lblk + map->m_len;
        depth = ext_depth(inode);
        ex = path[depth].p_ext;
        ee_block = le32_to_cpu(ex->ee_block);
        ee_len = ext4_ext_get_actual_len(ex);

        /* No split needed */
        if (ee_block == map->m_lblk && ee_len == map->m_len)
                goto convert;

        /*
         * It is only safe to convert extent to initialized via explicit
         * zeroout only if extent is fully inside i_size or new_size.
         */
        split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0;

        /*
         * pass SPLIT_NOMERGE explicitly so we don't end up merging extents we
         * just split.
         */
        path = ext4_split_extent(handle, inode, path, map, split_flag,
                                 flags | EXT4_GET_BLOCKS_SPLIT_NOMERGE,
                                 allocated, &did_zeroout);
        if (IS_ERR(path))
                return path;

convert:
        path = ext4_find_extent(inode, map->m_lblk, path, flags);
        if (IS_ERR(path))
                return path;

        depth = ext_depth(inode);
        ex = path[depth].p_ext;

        /*
         * Conversion is already handled in case of zeroout
         */
        if (!did_zeroout) {
                err = ext4_ext_get_access(handle, inode, path + depth);
                if (err)
                        goto err;

                if (flags & EXT4_GET_BLOCKS_CONVERT)
                        ext4_ext_mark_initialized(ex);
                else if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)
                        ext4_ext_mark_unwritten(ex);

                if (!(flags & EXT4_GET_BLOCKS_SPLIT_NOMERGE))
                       /*
                        * note: ext4_ext_correct_indexes() isn't needed here because
                        * borders are not changed
                        */
                        ext4_ext_try_to_merge(handle, inode, path, ex);

                err = ext4_ext_dirty(handle, inode, path + depth);
                if (err)
                        goto err;
        }

        /* Lets update the extent status tree after conversion */
        if (!(flags & EXT4_EX_NOCACHE))
                ext4_es_insert_extent(inode, le32_to_cpu(ex->ee_block),
                                      ext4_ext_get_actual_len(ex),
                                      ext4_ext_pblock(ex),
                                      ext4_ext_is_unwritten(ex) ?
                                              EXTENT_STATUS_UNWRITTEN :
                                              EXTENT_STATUS_WRITTEN,
                                      false);

err:
        if (err) {
                ext4_free_ext_path(path);
                return ERR_PTR(err);
        }

        return path;
}

static struct ext4_ext_path *
ext4_convert_unwritten_extents_endio(handle_t *handle, struct inode *inode,
                                     struct ext4_map_blocks *map,
                                     struct ext4_ext_path *path, int flags)
{
        struct ext4_extent *ex;
        ext4_lblk_t ee_block;
        unsigned int ee_len;
        int depth;

        depth = ext_depth(inode);
        ex = path[depth].p_ext;
        ee_block = le32_to_cpu(ex->ee_block);
        ee_len = ext4_ext_get_actual_len(ex);

        ext_debug(inode, "logical block %llu, max_blocks %u\n",
                  (unsigned long long)ee_block, ee_len);

        return ext4_split_convert_extents(handle, inode, map, path, flags,
                                          NULL);
}

static struct ext4_ext_path *
convert_initialized_extent(handle_t *handle, struct inode *inode,
                           struct ext4_map_blocks *map,
                           struct ext4_ext_path *path,
                           int flags,
                           unsigned int *allocated)
{
        struct ext4_extent *ex;
        ext4_lblk_t ee_block;
        unsigned int ee_len;
        int depth;

        /*
         * Make sure that the extent is no bigger than we support with
         * unwritten extent
         */
        if (map->m_len > EXT_UNWRITTEN_MAX_LEN)
                map->m_len = EXT_UNWRITTEN_MAX_LEN / 2;

        depth = ext_depth(inode);
        ex = path[depth].p_ext;
        ee_block = le32_to_cpu(ex->ee_block);
        ee_len = ext4_ext_get_actual_len(ex);

        ext_debug(inode, "logical block %llu, max_blocks %u\n",
                  (unsigned long long)ee_block, ee_len);

        path = ext4_split_convert_extents(handle, inode, map, path, flags,
                                          NULL);
        if (IS_ERR(path))
                return path;

        ext4_ext_show_leaf(inode, path);

        ext4_update_inode_fsync_trans(handle, inode, 1);

        /*
         * The extent might be initialized in case of zeroout.
         */
        path = ext4_find_extent(inode, map->m_lblk, path, flags);
        if (IS_ERR(path))
                return path;

        depth = ext_depth(inode);
        ex = path[depth].p_ext;

        if (ext4_ext_is_unwritten(ex))
                map->m_flags |= EXT4_MAP_UNWRITTEN;
        else
                map->m_flags |= EXT4_MAP_MAPPED;
        if (*allocated > map->m_len)
                *allocated = map->m_len;
        map->m_len = *allocated;
        return path;
}

static struct ext4_ext_path *
ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode,
                        struct ext4_map_blocks *map,
                        struct ext4_ext_path *path, int flags,
                        unsigned int *allocated, ext4_fsblk_t newblock)
{
        int err = 0;

        ext_debug(inode, "logical block %llu, max_blocks %u, flags 0x%x, allocated %u\n",
                  (unsigned long long)map->m_lblk, map->m_len, flags,
                  *allocated);
        ext4_ext_show_leaf(inode, path);

        /*
         * When writing into unwritten space, we should not fail to
         * allocate metadata blocks for the new extent block if needed.
         */
        flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL;

        trace_ext4_ext_handle_unwritten_extents(inode, map, flags,
                                                *allocated, newblock);

        /* IO end_io complete, convert the filled extent to written */
        if (flags & EXT4_GET_BLOCKS_CONVERT) {
                path = ext4_convert_unwritten_extents_endio(handle, inode,
                                                            map, path, flags);
                if (IS_ERR(path))
                        return path;
                ext4_update_inode_fsync_trans(handle, inode, 1);
                goto map_out;
        }
        /* buffered IO cases */
        /*
         * repeat fallocate creation request
         * we already have an unwritten extent
         */
        if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) {
                map->m_flags |= EXT4_MAP_UNWRITTEN;
                goto map_out;
        }

        /* buffered READ or buffered write_begin() lookup */
        if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
                /*
                 * We have blocks reserved already.  We
                 * return allocated blocks so that delalloc
                 * won't do block reservation for us.  But
                 * the buffer head will be unmapped so that
                 * a read from the block returns 0s.
                 */
                map->m_flags |= EXT4_MAP_UNWRITTEN;
                goto out1;
        }

        /*
         * Default case when (flags & EXT4_GET_BLOCKS_CREATE) == 1.
         * For buffered writes, at writepage time, etc.  Convert a
         * discovered unwritten extent to written.
         */
        path = ext4_ext_convert_to_initialized(handle, inode, map, path,
                                               flags, allocated);
        if (IS_ERR(path))
                return path;
        ext4_update_inode_fsync_trans(handle, inode, 1);
        /*
         * shouldn't get a 0 allocated when converting an unwritten extent
         * unless m_len is 0 (bug) or extent has been corrupted
         */
        if (unlikely(*allocated == 0)) {
                EXT4_ERROR_INODE(inode, "unexpected allocated == 0, m_len = %u",
                                 map->m_len);
                err = -EFSCORRUPTED;
                goto errout;
        }

        map->m_flags |= EXT4_MAP_NEW;
map_out:
        map->m_flags |= EXT4_MAP_MAPPED;
out1:
        map->m_pblk = newblock;
        if (*allocated > map->m_len)
                *allocated = map->m_len;
        map->m_len = *allocated;
        ext4_ext_show_leaf(inode, path);
        return path;

errout:
        ext4_free_ext_path(path);
        return ERR_PTR(err);
}

/*
 * get_implied_cluster_alloc - check to see if the requested
 * allocation (in the map structure) overlaps with a cluster already
 * allocated in an extent.
 *        @sb        The filesystem superblock structure
 *        @map        The requested lblk->pblk mapping
 *        @ex        The extent structure which might contain an implied
 *                        cluster allocation
 *
 * This function is called by ext4_ext_map_blocks() after we failed to
 * find blocks that were already in the inode's extent tree.  Hence,
 * we know that the beginning of the requested region cannot overlap
 * the extent from the inode's extent tree.  There are three cases we
 * want to catch.  The first is this case:
 *
 *                 |--- cluster # N--|
 *    |--- extent ---|        |---- requested region ---|
 *                        |==========|
 *
 * The second case that we need to test for is this one:
 *
 *   |--------- cluster # N ----------------|
 *           |--- requested region --|   |------- extent ----|
 *           |=======================|
 *
 * The third case is when the requested region lies between two extents
 * within the same cluster:
 *          |------------- cluster # N-------------|
 * |----- ex -----|                  |---- ex_right ----|
 *                  |------ requested region ------|
 *                  |================|
 *
 * In each of the above cases, we need to set the map->m_pblk and
 * map->m_len so it corresponds to the return the extent labelled as
 * "|====|" from cluster #N, since it is already in use for data in
 * cluster EXT4_B2C(sbi, map->m_lblk).        We will then return 1 to
 * signal to ext4_ext_map_blocks() that map->m_pblk should be treated
 * as a new "allocated" block region.  Otherwise, we will return 0 and
 * ext4_ext_map_blocks() will then allocate one or more new clusters
 * by calling ext4_mb_new_blocks().
 */
static int get_implied_cluster_alloc(struct super_block *sb,
                                     struct ext4_map_blocks *map,
                                     struct ext4_extent *ex,
                                     struct ext4_ext_path *path)
{
        struct ext4_sb_info *sbi = EXT4_SB(sb);
        ext4_lblk_t c_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
        ext4_lblk_t ex_cluster_start, ex_cluster_end;
        ext4_lblk_t rr_cluster_start;
        ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
        ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
        unsigned short ee_len = ext4_ext_get_actual_len(ex);

        /* The extent passed in that we are trying to match */
        ex_cluster_start = EXT4_B2C(sbi, ee_block);
        ex_cluster_end = EXT4_B2C(sbi, ee_block + ee_len - 1);

        /* The requested region passed into ext4_map_blocks() */
        rr_cluster_start = EXT4_B2C(sbi, map->m_lblk);

        if ((rr_cluster_start == ex_cluster_end) ||
            (rr_cluster_start == ex_cluster_start)) {
                if (rr_cluster_start == ex_cluster_end)
                        ee_start += ee_len - 1;
                map->m_pblk = EXT4_PBLK_CMASK(sbi, ee_start) + c_offset;
                map->m_len = min(map->m_len,
                                 (unsigned) sbi->s_cluster_ratio - c_offset);
                /*
                 * Check for and handle this case:
                 *
                 *   |--------- cluster # N-------------|
                 *                       |------- extent ----|
                 *           |--- requested region ---|
                 *           |===========|
                 */

                if (map->m_lblk < ee_block)
                        map->m_len = min(map->m_len, ee_block - map->m_lblk);

                /*
                 * Check for the case where there is already another allocated
                 * block to the right of 'ex' but before the end of the cluster.
                 *
                 *          |------------- cluster # N-------------|
                 * |----- ex -----|                  |---- ex_right ----|
                 *                  |------ requested region ------|
                 *                  |================|
                 */
                if (map->m_lblk > ee_block) {
                        ext4_lblk_t next = ext4_ext_next_allocated_block(path);
                        map->m_len = min(map->m_len, next - map->m_lblk);
                }

                trace_ext4_get_implied_cluster_alloc_exit(sb, map, 1);
                return 1;
        }

        trace_ext4_get_implied_cluster_alloc_exit(sb, map, 0);
        return 0;
}

/*
 * Determine hole length around the given logical block, first try to
 * locate and expand the hole from the given @path, and then adjust it
 * if it's partially or completely converted to delayed extents, insert
 * it into the extent cache tree if it's indeed a hole, finally return
 * the length of the determined extent.
 */
static ext4_lblk_t ext4_ext_determine_insert_hole(struct inode *inode,
                                                  struct ext4_ext_path *path,
                                                  ext4_lblk_t lblk)
{
        ext4_lblk_t hole_start, len;
        struct extent_status es;

        hole_start = lblk;
        len = ext4_ext_find_hole(inode, path, &hole_start);
again:
        ext4_es_find_extent_range(inode, &ext4_es_is_delayed, hole_start,
                                  hole_start + len - 1, &es);
        if (!es.es_len)
                goto insert_hole;

        /*
         * There's a delalloc extent in the hole, handle it if the delalloc
         * extent is in front of, behind and straddle the queried range.
         */
        if (lblk >= es.es_lblk + es.es_len) {
                /*
                 * The delalloc extent is in front of the queried range,
                 * find again from the queried start block.
                 */
                len -= lblk - hole_start;
                hole_start = lblk;
                goto again;
        } else if (in_range(lblk, es.es_lblk, es.es_len)) {
                /*
                 * The delalloc extent containing lblk, it must have been
                 * added after ext4_map_blocks() checked the extent status
                 * tree so we are not holding i_rwsem and delalloc info is
                 * only stabilized by i_data_sem we are going to release
                 * soon. Don't modify the extent status tree and report
                 * extent as a hole, just adjust the length to the delalloc
                 * extent's after lblk.
                 */
                len = es.es_lblk + es.es_len - lblk;
                return len;
        } else {
                /*
                 * The delalloc extent is partially or completely behind
                 * the queried range, update hole length until the
                 * beginning of the delalloc extent.
                 */
                len = min(es.es_lblk - hole_start, len);
        }

insert_hole:
        /* Put just found gap into cache to speed up subsequent requests */
        ext_debug(inode, " -> %u:%u\n", hole_start, len);
        ext4_es_cache_extent(inode, hole_start, len, ~0, EXTENT_STATUS_HOLE);

        /* Update hole_len to reflect hole size after lblk */
        if (hole_start != lblk)
                len -= lblk - hole_start;

        return len;
}

/*
 * Block allocation/map/preallocation routine for extents based files
 *
 *
 * Need to be called with
 * down_read(&EXT4_I(inode)->i_data_sem) if not allocating file system block
 * (ie, flags is zero). Otherwise down_write(&EXT4_I(inode)->i_data_sem)
 *
 * return > 0, number of blocks already mapped/allocated
 *          if flags doesn't contain EXT4_GET_BLOCKS_CREATE and these are pre-allocated blocks
 *                  buffer head is unmapped
 *          otherwise blocks are mapped
 *
 * return = 0, if plain look up failed (blocks have not been allocated)
 *          buffer head is unmapped
 *
 * return < 0, error case.
 */
int ext4_ext_map_blocks(handle_t *handle, struct inode *inode,
                        struct ext4_map_blocks *map, int flags)
{
        struct ext4_ext_path *path = NULL;
        struct ext4_extent newex, *ex, ex2;
        struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
        ext4_fsblk_t newblock = 0, pblk;
        int err = 0, depth;
        unsigned int allocated = 0, offset = 0;
        unsigned int allocated_clusters = 0;
        struct ext4_allocation_request ar;
        ext4_lblk_t cluster_offset;

        ext_debug(inode, "blocks %u/%u requested\n", map->m_lblk, map->m_len);
        trace_ext4_ext_map_blocks_enter(inode, map->m_lblk, map->m_len, flags);

        /* find extent for this block */
        path = ext4_find_extent(inode, map->m_lblk, NULL, flags);
        if (IS_ERR(path)) {
                err = PTR_ERR(path);
                goto out;
        }

        depth = ext_depth(inode);

        /*
         * consistent leaf must not be empty;
         * this situation is possible, though, _during_ tree modification;
         * this is why assert can't be put in ext4_find_extent()
         */
        if (unlikely(path[depth].p_ext == NULL && depth != 0)) {
                EXT4_ERROR_INODE(inode, "bad extent address "
                                 "lblock: %lu, depth: %d pblock %lld",
                                 (unsigned long) map->m_lblk, depth,
                                 path[depth].p_block);
                err = -EFSCORRUPTED;
                goto out;
        }

        ex = path[depth].p_ext;
        if (ex) {
                ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
                ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
                unsigned short ee_len;


                /*
                 * unwritten extents are treated as holes, except that
                 * we split out initialized portions during a write.
                 */
                ee_len = ext4_ext_get_actual_len(ex);

                trace_ext4_ext_show_extent(inode, ee_block, ee_start, ee_len);

                /* if found extent covers block, simply return it */
                if (in_range(map->m_lblk, ee_block, ee_len)) {
                        newblock = map->m_lblk - ee_block + ee_start;
                        /* number of remaining blocks in the extent */
                        allocated = ee_len - (map->m_lblk - ee_block);
                        ext_debug(inode, "%u fit into %u:%d -> %llu\n",
                                  map->m_lblk, ee_block, ee_len, newblock);

                        /*
                         * If the extent is initialized check whether the
                         * caller wants to convert it to unwritten.
                         */
                        if ((!ext4_ext_is_unwritten(ex)) &&
                            (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) {
                                path = convert_initialized_extent(handle,
                                        inode, map, path, flags, &allocated);
                                if (IS_ERR(path))
                                        err = PTR_ERR(path);
                                goto out;
                        } else if (!ext4_ext_is_unwritten(ex)) {
                                map->m_flags |= EXT4_MAP_MAPPED;
                                map->m_pblk = newblock;
                                if (allocated > map->m_len)
                                        allocated = map->m_len;
                                map->m_len = allocated;
                                ext4_ext_show_leaf(inode, path);
                                goto out;
                        }

                        path = ext4_ext_handle_unwritten_extents(
                                handle, inode, map, path, flags,
                                &allocated, newblock);
                        if (IS_ERR(path))
                                err = PTR_ERR(path);
                        goto out;
                }
        }

        /*
         * requested block isn't allocated yet;
         * we couldn't try to create block if flags doesn't contain EXT4_GET_BLOCKS_CREATE
         */
        if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
                ext4_lblk_t len;

                len = ext4_ext_determine_insert_hole(inode, path, map->m_lblk);

                map->m_pblk = 0;
                map->m_len = min_t(unsigned int, map->m_len, len);
                goto out;
        }

        /*
         * Okay, we need to do block allocation.
         */
        newex.ee_block = cpu_to_le32(map->m_lblk);
        cluster_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);

        /*
         * If we are doing bigalloc, check to see if the extent returned
         * by ext4_find_extent() implies a cluster we can use.
         */
        if (cluster_offset && ex &&
            get_implied_cluster_alloc(inode->i_sb, map, ex, path)) {
                ar.len = allocated = map->m_len;
                newblock = map->m_pblk;
                goto got_allocated_blocks;
        }

        /* find neighbour allocated blocks */
        ar.lleft = map->m_lblk;
        err = ext4_ext_search_left(inode, path, &ar.lleft, &ar.pleft);
        if (err)
                goto out;
        ar.lright = map->m_lblk;
        err = ext4_ext_search_right(inode, path, &ar.lright, &ar.pright,
                                    &ex2, flags);
        if (err < 0)
                goto out;

        /* Check if the extent after searching to the right implies a
         * cluster we can use. */
        if ((sbi->s_cluster_ratio > 1) && err &&
            get_implied_cluster_alloc(inode->i_sb, map, &ex2, path)) {
                ar.len = allocated = map->m_len;
                newblock = map->m_pblk;
                err = 0;
                goto got_allocated_blocks;
        }

        /*
         * See if request is beyond maximum number of blocks we can have in
         * a single extent. For an initialized extent this limit is
         * EXT_INIT_MAX_LEN and for an unwritten extent this limit is
         * EXT_UNWRITTEN_MAX_LEN.
         */
        if (map->m_len > EXT_INIT_MAX_LEN &&
            !(flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
                map->m_len = EXT_INIT_MAX_LEN;
        else if (map->m_len > EXT_UNWRITTEN_MAX_LEN &&
                 (flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
                map->m_len = EXT_UNWRITTEN_MAX_LEN;

        /* Check if we can really insert (m_lblk)::(m_lblk + m_len) extent */
        newex.ee_len = cpu_to_le16(map->m_len);
        err = ext4_ext_check_overlap(sbi, inode, &newex, path);
        if (err)
                allocated = ext4_ext_get_actual_len(&newex);
        else
                allocated = map->m_len;

        /* allocate new block */
        ar.inode = inode;
        ar.goal = ext4_ext_find_goal(inode, path, map->m_lblk);
        ar.logical = map->m_lblk;
        /*
         * We calculate the offset from the beginning of the cluster
         * for the logical block number, since when we allocate a
         * physical cluster, the physical block should start at the
         * same offset from the beginning of the cluster.  This is
         * needed so that future calls to get_implied_cluster_alloc()
         * work correctly.
         */
        offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
        ar.len = EXT4_NUM_B2C(sbi, offset+allocated);
        ar.goal -= offset;
        ar.logical -= offset;
        if (S_ISREG(inode->i_mode))
                ar.flags = EXT4_MB_HINT_DATA;
        else
                /* disable in-core preallocation for non-regular files */
                ar.flags = 0;
        if (flags & EXT4_GET_BLOCKS_NO_NORMALIZE)
                ar.flags |= EXT4_MB_HINT_NOPREALLOC;
        if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
                ar.flags |= EXT4_MB_DELALLOC_RESERVED;
        if (flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
                ar.flags |= EXT4_MB_USE_RESERVED;
        newblock = ext4_mb_new_blocks(handle, &ar, &err);
        if (!newblock)
                goto out;
        allocated_clusters = ar.len;
        ar.len = EXT4_C2B(sbi, ar.len) - offset;
        ext_debug(inode, "allocate new block: goal %llu, found %llu/%u, requested %u\n",
                  ar.goal, newblock, ar.len, allocated);
        if (ar.len > allocated)
                ar.len = allocated;

got_allocated_blocks:
        /* try to insert new extent into found leaf and return */
        pblk = newblock + offset;
        ext4_ext_store_pblock(&newex, pblk);
        newex.ee_len = cpu_to_le16(ar.len);
        /* Mark unwritten */
        if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) {
                ext4_ext_mark_unwritten(&newex);
                map->m_flags |= EXT4_MAP_UNWRITTEN;
        }

        path = ext4_ext_insert_extent(handle, inode, path, &newex, flags);
        if (IS_ERR(path)) {
                err = PTR_ERR(path);
                /*
                 * Gracefully handle out of space conditions. If the filesystem
                 * is inconsistent, we'll just leak allocated blocks to avoid
                 * causing even more damage.
                 */
                if (allocated_clusters && (err == -EDQUOT || err == -ENOSPC)) {
                        int fb_flags = 0;
                        /*
                         * free data blocks we just allocated.
                         * not a good idea to call discard here directly,
                         * but otherwise we'd need to call it every free().
                         */
                        ext4_discard_preallocations(inode);
                        if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
                                fb_flags = EXT4_FREE_BLOCKS_NO_QUOT_UPDATE;
                        ext4_free_blocks(handle, inode, NULL, newblock,
                                         EXT4_C2B(sbi, allocated_clusters),
                                         fb_flags);
                }
                goto out;
        }

        /*
         * Cache the extent and update transaction to commit on fdatasync only
         * when it is _not_ an unwritten extent.
         */
        if ((flags & EXT4_GET_BLOCKS_UNWRIT_EXT) == 0)
                ext4_update_inode_fsync_trans(handle, inode, 1);
        else
                ext4_update_inode_fsync_trans(handle, inode, 0);

        map->m_flags |= (EXT4_MAP_NEW | EXT4_MAP_MAPPED);
        map->m_pblk = pblk;
        map->m_len = ar.len;
        allocated = map->m_len;
        ext4_ext_show_leaf(inode, path);
out:
        /*
         * We never use EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF with CREATE flag.
         * So we know that the depth used here is correct, since there was no
         * block allocation done if EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF is set.
         * If tomorrow we start using this QUERY flag with CREATE, then we will
         * need to re-calculate the depth as it might have changed due to block
         * allocation.
         */
        if (flags & EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF) {
                WARN_ON_ONCE(flags & EXT4_GET_BLOCKS_CREATE);
                if (!err && ex && (ex == EXT_LAST_EXTENT(path[depth].p_hdr)))
                        map->m_flags |= EXT4_MAP_QUERY_LAST_IN_LEAF;
        }

        ext4_free_ext_path(path);

        trace_ext4_ext_map_blocks_exit(inode, flags, map,
                                       err ? err : allocated);
        return err ? err : allocated;
}

int ext4_ext_truncate(handle_t *handle, struct inode *inode)
{
        struct super_block *sb = inode->i_sb;
        ext4_lblk_t last_block;
        int err = 0;

        /*
         * TODO: optimization is possible here.
         * Probably we need not scan at all,
         * because page truncation is enough.
         */

        /* we have to know where to truncate from in crash case */
        EXT4_I(inode)->i_disksize = inode->i_size;
        err = ext4_mark_inode_dirty(handle, inode);
        if (err)
                return err;

        last_block = (inode->i_size + sb->s_blocksize - 1)
                        >> EXT4_BLOCK_SIZE_BITS(sb);
        ext4_es_remove_extent(inode, last_block, EXT_MAX_BLOCKS - last_block);

retry_remove_space:
        err = ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1);
        if (err == -ENOMEM) {
                memalloc_retry_wait(GFP_ATOMIC);
                goto retry_remove_space;
        }
        return err;
}

static int ext4_alloc_file_blocks(struct file *file, loff_t offset, loff_t len,
                                  loff_t new_size, int flags)
{
        struct inode *inode = file_inode(file);
        handle_t *handle;
        int ret = 0, ret2 = 0, ret3 = 0;
        int retries = 0;
        int depth = 0;
        ext4_lblk_t len_lblk;
        struct ext4_map_blocks map;
        unsigned int credits;
        loff_t epos = 0, old_size = i_size_read(inode);
        unsigned int blkbits = inode->i_blkbits;
        bool alloc_zero = false;

        BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS));
        map.m_lblk = offset >> blkbits;
        map.m_len = len_lblk = EXT4_MAX_BLOCKS(len, offset, blkbits);
        /*
         * Don't normalize the request if it can fit in one extent so
         * that it doesn't get unnecessarily split into multiple
         * extents.
         */
        if (len_lblk <= EXT_UNWRITTEN_MAX_LEN)
                flags |= EXT4_GET_BLOCKS_NO_NORMALIZE;

        /*
         * Do the actual write zero during a running journal transaction
         * costs a lot. First allocate an unwritten extent and then
         * convert it to written after zeroing it out.
         */
        if (flags & EXT4_GET_BLOCKS_ZERO) {
                flags &= ~EXT4_GET_BLOCKS_ZERO;
                flags |= EXT4_GET_BLOCKS_UNWRIT_EXT;
                alloc_zero = true;
        }

        /*
         * credits to insert 1 extent into extent tree
         */
        credits = ext4_chunk_trans_blocks(inode, len_lblk);
        depth = ext_depth(inode);

        /* Zero to the end of the block containing i_size */
        if (new_size > old_size) {
                ret = ext4_block_zero_eof(inode, old_size, LLONG_MAX);
                if (ret)
                        return ret;
        }

retry:
        while (len_lblk) {
                /*
                 * Recalculate credits when extent tree depth changes.
                 */
                if (depth != ext_depth(inode)) {
                        credits = ext4_chunk_trans_blocks(inode, len_lblk);
                        depth = ext_depth(inode);
                }

                handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
                                            credits);
                if (IS_ERR(handle)) {
                        ret = PTR_ERR(handle);
                        break;
                }
                ret = ext4_map_blocks(handle, inode, &map, flags);
                if (ret <= 0) {
                        ext4_debug("inode #%llu: block %u: len %u: "
                                   "ext4_ext_map_blocks returned %d",
                                   inode->i_ino, map.m_lblk,
                                   map.m_len, ret);
                        ext4_mark_inode_dirty(handle, inode);
                        ext4_journal_stop(handle);
                        break;
                }
                ext4_update_inode_fsync_trans(handle, inode, 1);
                ret = ext4_journal_stop(handle);
                if (unlikely(ret))
                        break;

                /*
                 * allow a full retry cycle for any remaining allocations
                 */
                retries = 0;

                if (alloc_zero &&
                    (map.m_flags & (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN))) {
                        ret = ext4_issue_zeroout(inode, map.m_lblk, map.m_pblk,
                                                 map.m_len);
                        if (likely(!ret))
                                ret = ext4_convert_unwritten_extents(NULL,
                                        inode, (loff_t)map.m_lblk << blkbits,
                                        (loff_t)map.m_len << blkbits);
                        if (ret)
                                break;
                }

                map.m_lblk += map.m_len;
                map.m_len = len_lblk = len_lblk - map.m_len;
                epos = EXT4_LBLK_TO_B(inode, map.m_lblk);
        }

        if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
                goto retry;

        if (!epos || !new_size)
                return ret;

        /*
         * Allocate blocks, update the file size to match the size of the
         * already successfully allocated blocks.
         */
        if (epos > new_size)
                epos = new_size;

        handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
        if (IS_ERR(handle))
                return ret ? ret : PTR_ERR(handle);

        ext4_update_inode_size(inode, epos);
        ret2 = ext4_mark_inode_dirty(handle, inode);
        ext4_update_inode_fsync_trans(handle, inode, 1);
        ret3 = ext4_journal_stop(handle);
        ret2 = ret3 ? ret3 : ret2;

        if (epos > old_size)
                pagecache_isize_extended(inode, old_size, epos);

        return ret ? ret : ret2;
}

static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len);

static int ext4_insert_range(struct file *file, loff_t offset, loff_t len);

static long ext4_zero_range(struct file *file, loff_t offset,
                            loff_t len, int mode)
{
        struct inode *inode = file_inode(file);
        handle_t *handle = NULL;
        loff_t align_start, align_end, new_size = 0;
        loff_t end = offset + len;
        unsigned int blocksize = i_blocksize(inode);
        bool partial_zeroed = false;
        int ret, flags;

        trace_ext4_zero_range(inode, offset, len, mode);
        WARN_ON_ONCE(!inode_is_locked(inode));

        /* Indirect files do not support unwritten extents */
        if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
                return -EOPNOTSUPP;

        if (!(mode & FALLOC_FL_KEEP_SIZE) &&
            (end > inode->i_size || end > EXT4_I(inode)->i_disksize)) {
                new_size = end;
                ret = inode_newsize_ok(inode, new_size);
                if (ret)
                        return ret;
        }

        flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
        /* Preallocate the range including the unaligned edges */
        if (!IS_ALIGNED(offset | end, blocksize)) {
                ret = ext4_alloc_file_blocks(file, offset, len, new_size,
                                             flags);
                if (ret)
                        return ret;
        }

        ret = ext4_update_disksize_before_punch(inode, offset, len);
        if (ret)
                return ret;

        /* Now release the pages and zero block aligned part of pages */
        ret = ext4_truncate_page_cache_block_range(inode, offset, end);
        if (ret)
                return ret;

        /* Zero range excluding the unaligned edges */
        align_start = round_up(offset, blocksize);
        align_end = round_down(end, blocksize);
        if (align_end > align_start) {
                if (mode & FALLOC_FL_WRITE_ZEROES)
                        flags = EXT4_GET_BLOCKS_CREATE_ZERO | EXT4_EX_NOCACHE;
                else
                        flags |= (EXT4_GET_BLOCKS_CONVERT_UNWRITTEN |
                                  EXT4_EX_NOCACHE);
                ret = ext4_alloc_file_blocks(file, align_start,
                                             align_end - align_start, new_size,
                                             flags);
                if (ret)
                        return ret;
        }
        /* Finish zeroing out if it doesn't contain partial block */
        if (IS_ALIGNED(offset | end, blocksize))
                return ret;

        /* Zero out partial block at the edges of the range */
        ret = ext4_zero_partial_blocks(inode, offset, len, &partial_zeroed);
        if (ret)
                return ret;
        if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed) {
                ret = filemap_write_and_wait_range(inode->i_mapping, offset,
                                                   end - 1);
                if (ret)
                        return ret;
        }

        handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
        if (IS_ERR(handle)) {
                ret = PTR_ERR(handle);
                ext4_std_error(inode->i_sb, ret);
                return ret;
        }

        if (new_size)
                ext4_update_inode_size(inode, new_size);
        ret = ext4_mark_inode_dirty(handle, inode);
        if (unlikely(ret))
                goto out_handle;

        ext4_update_inode_fsync_trans(handle, inode, 1);
        if ((file->f_flags & O_SYNC) || IS_SYNC(inode))
                ext4_handle_sync(handle);

out_handle:
        ext4_journal_stop(handle);
        return ret;
}

static long ext4_do_fallocate(struct file *file, loff_t offset,
                              loff_t len, int mode)
{
        struct inode *inode = file_inode(file);
        loff_t end = offset + len;
        loff_t new_size = 0;
        int ret;

        trace_ext4_fallocate_enter(inode, offset, len, mode);
        WARN_ON_ONCE(!inode_is_locked(inode));

        /* We only support preallocation for extent-based files only. */
        if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
                ret = -EOPNOTSUPP;
                goto out;
        }

        if (!(mode & FALLOC_FL_KEEP_SIZE) &&
            (end > inode->i_size || end > EXT4_I(inode)->i_disksize)) {
                new_size = end;
                ret = inode_newsize_ok(inode, new_size);
                if (ret)
                        goto out;
        }

        ret = ext4_alloc_file_blocks(file, offset, len, new_size,
                                     EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT);
        if (ret)
                goto out;

        if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) &&
            EXT4_SB(inode->i_sb)->s_journal) {
                ret = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
                                        EXT4_I(inode)->i_sync_tid);
        }
out:
        trace_ext4_fallocate_exit(inode, offset,
                        EXT4_MAX_BLOCKS(len, offset, inode->i_blkbits), ret);
        return ret;
}

/*
 * preallocate space for a file. This implements ext4's fallocate file
 * operation, which gets called from sys_fallocate system call.
 * For block-mapped files, posix_fallocate should fall back to the method
 * of writing zeroes to the required new blocks (the same behavior which is
 * expected for file systems which do not support fallocate() system call).
 */
long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
{
        struct inode *inode = file_inode(file);
        struct address_space *mapping = file->f_mapping;
        int ret;

        /*
         * Encrypted inodes can't handle collapse range or insert
         * range since we would need to re-encrypt blocks with a
         * different IV or XTS tweak (which are based on the logical
         * block number).
         */
        if (IS_ENCRYPTED(inode) &&
            (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE)))
                return -EOPNOTSUPP;
        /*
         * Don't allow writing zeroes if the underlying device does not
         * enable the unmap write zeroes operation.
         */
        if ((mode & FALLOC_FL_WRITE_ZEROES) &&
            !bdev_write_zeroes_unmap_sectors(inode->i_sb->s_bdev))
                return -EOPNOTSUPP;

        /* Return error if mode is not supported */
        if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
                     FALLOC_FL_ZERO_RANGE | FALLOC_FL_COLLAPSE_RANGE |
                     FALLOC_FL_INSERT_RANGE | FALLOC_FL_WRITE_ZEROES))
                return -EOPNOTSUPP;

        inode_lock(inode);
        ret = ext4_convert_inline_data(inode);
        if (ret)
                goto out_inode_lock;

        /* Wait all existing dio workers, newcomers will block on i_rwsem */
        inode_dio_wait(inode);

        ret = file_modified(file);
        if (ret)
                goto out_inode_lock;

        if ((mode & FALLOC_FL_MODE_MASK) == FALLOC_FL_ALLOCATE_RANGE) {
                ret = ext4_do_fallocate(file, offset, len, mode);
                goto out_inode_lock;
        }

        /*
         * Follow-up operations will drop page cache, hold invalidate lock
         * to prevent page faults from reinstantiating pages we have
         * released from page cache.
         */
        filemap_invalidate_lock(mapping);

        ret = ext4_break_layouts(inode);
        if (ret)
                goto out_invalidate_lock;

        switch (mode & FALLOC_FL_MODE_MASK) {
        case FALLOC_FL_PUNCH_HOLE:
                ret = ext4_punch_hole(file, offset, len);
                break;
        case FALLOC_FL_COLLAPSE_RANGE:
                ret = ext4_collapse_range(file, offset, len);
                break;
        case FALLOC_FL_INSERT_RANGE:
                ret = ext4_insert_range(file, offset, len);
                break;
        case FALLOC_FL_ZERO_RANGE:
        case FALLOC_FL_WRITE_ZEROES:
                ret = ext4_zero_range(file, offset, len, mode);
                break;
        default:
                ret = -EOPNOTSUPP;
        }

out_invalidate_lock:
        filemap_invalidate_unlock(mapping);
out_inode_lock:
        inode_unlock(inode);
        return ret;
}

/*
 * This function converts a range of blocks to written extents. The caller of
 * this function will pass the start offset and the size. all unwritten extents
 * within this range will be converted to written extents.
 *
 * This function is called from the direct IO end io call back function for
 * atomic writes, to convert the unwritten extents after IO is completed.
 *
 * Note that the requirement for atomic writes is that all conversion should
 * happen atomically in a single fs journal transaction. We mainly only allocate
 * unwritten extents either on a hole on a pre-exiting unwritten extent range in
 * ext4_map_blocks_atomic_write(). The only case where we can have multiple
 * unwritten extents in a range [offset, offset+len) is when there is a split
 * unwritten extent between two leaf nodes which was cached in extent status
 * cache during ext4_iomap_alloc() time. That will allow
 * ext4_map_blocks_atomic_write() to return the unwritten extent range w/o going
 * into the slow path. That means we might need a loop for conversion of this
 * unwritten extent split across leaf block within a single journal transaction.
 * Split extents across leaf nodes is a rare case, but let's still handle that
 * to meet the requirements of multi-fsblock atomic writes.
 *
 * Returns 0 on success.
 */
int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode,
                                          loff_t offset, ssize_t len)
{
        unsigned int max_blocks;
        int ret = 0, ret2 = 0, ret3 = 0;
        struct ext4_map_blocks map;
        unsigned int blkbits = inode->i_blkbits;
        unsigned int credits = 0;
        int flags = EXT4_GET_BLOCKS_IO_CONVERT_EXT | EXT4_EX_NOCACHE;

        map.m_lblk = offset >> blkbits;
        max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);

        if (!handle) {
                /*
                 * TODO: An optimization can be added later by having an extent
                 * status flag e.g. EXTENT_STATUS_SPLIT_LEAF. If we query that
                 * it can tell if the extent in the cache is a split extent.
                 * But for now let's assume pextents as 2 always.
                 */
                credits = ext4_meta_trans_blocks(inode, max_blocks, 2);
        }

        if (credits) {
                handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, credits);
                if (IS_ERR(handle)) {
                        ret = PTR_ERR(handle);
                        return ret;
                }
        }

        while (ret >= 0 && ret < max_blocks) {
                map.m_lblk += ret;
                map.m_len = (max_blocks -= ret);
                ret = ext4_map_blocks(handle, inode, &map, flags);
                if (ret != max_blocks)
                        ext4_msg(inode->i_sb, KERN_INFO,
                                     "inode #%llu: block %u: len %u: "
                                     "split block mapping found for atomic write, "
                                     "ret = %d",
                                     inode->i_ino, map.m_lblk,
                                     map.m_len, ret);
                if (ret <= 0)
                        break;
        }

        ret2 = ext4_mark_inode_dirty(handle, inode);

        if (credits) {
                ret3 = ext4_journal_stop(handle);
                if (unlikely(ret3))
                        ret2 = ret3;
        }

        if (ret <= 0 || ret2)
                ext4_warning(inode->i_sb,
                             "inode #%llu: block %u: len %u: "
                             "returned %d or %d",
                             inode->i_ino, map.m_lblk,
                             map.m_len, ret, ret2);

        return ret > 0 ? ret2 : ret;
}

/*
 * This function convert a range of blocks to written extents
 * The caller of this function will pass the start offset and the size.
 * all unwritten extents within this range will be converted to
 * written extents.
 *
 * This function is called from the direct IO end io call back
 * function, to convert the fallocated extents after IO is completed.
 * Returns 0 on success.
 */
int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
                                   loff_t offset, ssize_t len)
{
        unsigned int max_blocks;
        int ret = 0, ret2 = 0, ret3 = 0;
        struct ext4_map_blocks map;
        unsigned int blkbits = inode->i_blkbits;
        unsigned int credits = 0;

        map.m_lblk = offset >> blkbits;
        max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);

        if (!handle) {
                /*
                 * credits to insert 1 extent into extent tree
                 */
                credits = ext4_chunk_trans_blocks(inode, max_blocks);
        }
        while (ret >= 0 && ret < max_blocks) {
                map.m_lblk += ret;
                map.m_len = (max_blocks -= ret);
                if (credits) {
                        handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
                                                    credits);
                        if (IS_ERR(handle)) {
                                ret = PTR_ERR(handle);
                                break;
                        }
                }
                /*
                 * Do not cache any unrelated extents, as it does not hold the
                 * i_rwsem or invalidate_lock, which could corrupt the extent
                 * status tree.
                 */
                ret = ext4_map_blocks(handle, inode, &map,
                                      EXT4_GET_BLOCKS_IO_CONVERT_EXT |
                                      EXT4_EX_NOCACHE);
                if (ret <= 0)
                        ext4_warning(inode->i_sb,
                                     "inode #%llu: block %u: len %u: "
                                     "ext4_ext_map_blocks returned %d",
                                     inode->i_ino, map.m_lblk,
                                     map.m_len, ret);
                ret2 = ext4_mark_inode_dirty(handle, inode);
                if (credits) {
                        ret3 = ext4_journal_stop(handle);
                        if (unlikely(ret3))
                                ret2 = ret3;
                }

                if (ret <= 0 || ret2)
                        break;
        }
        return ret > 0 ? ret2 : ret;
}

int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end)
{
        int ret = 0, err = 0;
        struct ext4_io_end_vec *io_end_vec;

        /*
         * This is somewhat ugly but the idea is clear: When transaction is
         * reserved, everything goes into it. Otherwise we rather start several
         * smaller transactions for conversion of each extent separately.
         */
        if (handle) {
                handle = ext4_journal_start_reserved(handle,
                                                     EXT4_HT_EXT_CONVERT);
                if (IS_ERR(handle))
                        return PTR_ERR(handle);
        }

        list_for_each_entry(io_end_vec, &io_end->list_vec, list) {
                ret = ext4_convert_unwritten_extents(handle, io_end->inode,
                                                     io_end_vec->offset,
                                                     io_end_vec->size);
                if (ret)
                        break;
        }

        if (handle)
                err = ext4_journal_stop(handle);

        return ret < 0 ? ret : err;
}

static int ext4_iomap_xattr_fiemap(struct inode *inode, struct iomap *iomap)
{
        __u64 physical = 0;
        __u64 length = 0;
        int blockbits = inode->i_sb->s_blocksize_bits;
        int error = 0;
        u16 iomap_type;

        /* in-inode? */
        if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
                struct ext4_iloc iloc;
                int offset;        /* offset of xattr in inode */

                error = ext4_get_inode_loc(inode, &iloc);
                if (error)
                        return error;
                physical = (__u64)iloc.bh->b_blocknr << blockbits;
                offset = EXT4_GOOD_OLD_INODE_SIZE +
                                EXT4_I(inode)->i_extra_isize;
                physical += offset;
                length = EXT4_SB(inode->i_sb)->s_inode_size - offset;
                brelse(iloc.bh);
                iomap_type = IOMAP_INLINE;
        } else if (EXT4_I(inode)->i_file_acl) { /* external block */
                physical = (__u64)EXT4_I(inode)->i_file_acl << blockbits;
                length = inode->i_sb->s_blocksize;
                iomap_type = IOMAP_MAPPED;
        } else {
                /* no in-inode or external block for xattr, so return -ENOENT */
                error = -ENOENT;
                goto out;
        }

        iomap->addr = physical;
        iomap->offset = 0;
        iomap->length = length;
        iomap->type = iomap_type;
        iomap->flags = 0;
out:
        return error;
}

static int ext4_iomap_xattr_begin(struct inode *inode, loff_t offset,
                                  loff_t length, unsigned flags,
                                  struct iomap *iomap, struct iomap *srcmap)
{
        int error;

        error = ext4_iomap_xattr_fiemap(inode, iomap);
        if (error == 0 && (offset >= iomap->length))
                error = -ENOENT;
        return error;
}

static const struct iomap_ops ext4_iomap_xattr_ops = {
        .iomap_begin                = ext4_iomap_xattr_begin,
};

static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len)
{
        u64 maxbytes = ext4_get_maxbytes(inode);

        if (*len == 0)
                return -EINVAL;
        if (start > maxbytes)
                return -EFBIG;

        /*
         * Shrink request scope to what the fs can actually handle.
         */
        if (*len > maxbytes || (maxbytes - *len) < start)
                *len = maxbytes - start;
        return 0;
}

int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
                u64 start, u64 len)
{
        int error = 0;

        inode_lock_shared(inode);
        if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
                error = ext4_ext_precache(inode);
                if (error)
                        goto unlock;
                fieinfo->fi_flags &= ~FIEMAP_FLAG_CACHE;
        }

        /*
         * For bitmap files the maximum size limit could be smaller than
         * s_maxbytes, so check len here manually instead of just relying on the
         * generic check.
         */
        error = ext4_fiemap_check_ranges(inode, start, &len);
        if (error)
                goto unlock;

        if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
                fieinfo->fi_flags &= ~FIEMAP_FLAG_XATTR;
                error = iomap_fiemap(inode, fieinfo, start, len,
                                     &ext4_iomap_xattr_ops);
        } else {
                error = iomap_fiemap(inode, fieinfo, start, len,
                                     &ext4_iomap_report_ops);
        }
unlock:
        inode_unlock_shared(inode);
        return error;
}

int ext4_get_es_cache(struct inode *inode, struct fiemap_extent_info *fieinfo,
                      __u64 start, __u64 len)
{
        ext4_lblk_t start_blk, len_blks;
        __u64 last_blk;
        int error = 0;

        if (ext4_has_inline_data(inode)) {
                int has_inline;

                down_read(&EXT4_I(inode)->xattr_sem);
                has_inline = ext4_has_inline_data(inode);
                up_read(&EXT4_I(inode)->xattr_sem);
                if (has_inline)
                        return 0;
        }

        if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
                inode_lock_shared(inode);
                error = ext4_ext_precache(inode);
                inode_unlock_shared(inode);
                if (error)
                        return error;
                fieinfo->fi_flags &= ~FIEMAP_FLAG_CACHE;
        }

        error = fiemap_prep(inode, fieinfo, start, &len, 0);
        if (error)
                return error;

        error = ext4_fiemap_check_ranges(inode, start, &len);
        if (error)
                return error;

        start_blk = start >> inode->i_sb->s_blocksize_bits;
        last_blk = (start + len - 1) >> inode->i_sb->s_blocksize_bits;
        if (last_blk >= EXT_MAX_BLOCKS)
                last_blk = EXT_MAX_BLOCKS-1;
        len_blks = ((ext4_lblk_t) last_blk) - start_blk + 1;

        /*
         * Walk the extent tree gathering extent information
         * and pushing extents back to the user.
         */
        return ext4_fill_es_cache_info(inode, start_blk, len_blks, fieinfo);
}

/*
 * ext4_ext_shift_path_extents:
 * Shift the extents of a path structure lying between path[depth].p_ext
 * and EXT_LAST_EXTENT(path[depth].p_hdr), by @shift blocks. @SHIFT tells
 * if it is right shift or left shift operation.
 */
static int
ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift,
                            struct inode *inode, handle_t *handle,
                            enum SHIFT_DIRECTION SHIFT)
{
        int depth, err = 0;
        struct ext4_extent *ex_start, *ex_last;
        bool update = false;
        int credits, restart_credits;
        depth = path->p_depth;

        while (depth >= 0) {
                if (depth == path->p_depth) {
                        ex_start = path[depth].p_ext;
                        if (!ex_start)
                                return -EFSCORRUPTED;

                        ex_last = EXT_LAST_EXTENT(path[depth].p_hdr);
                        /* leaf + sb + inode */
                        credits = 3;
                        if (ex_start == EXT_FIRST_EXTENT(path[depth].p_hdr)) {
                                update = true;
                                /* extent tree + sb + inode */
                                credits = depth + 2;
                        }

                        restart_credits = ext4_chunk_trans_extent(inode, 0);
                        err = ext4_datasem_ensure_credits(handle, inode, credits,
                                        restart_credits, 0);
                        if (err) {
                                if (err > 0)
                                        err = -EAGAIN;
                                goto out;
                        }

                        err = ext4_ext_get_access(handle, inode, path + depth);
                        if (err)
                                goto out;

                        while (ex_start <= ex_last) {
                                if (SHIFT == SHIFT_LEFT) {
                                        le32_add_cpu(&ex_start->ee_block,
                                                -shift);
                                        /* Try to merge to the left. */
                                        if ((ex_start >
                                            EXT_FIRST_EXTENT(path[depth].p_hdr))
                                            &&
                                            ext4_ext_try_to_merge_right(inode,
                                            path, ex_start - 1))
                                                ex_last--;
                                        else
                                                ex_start++;
                                } else {
                                        le32_add_cpu(&ex_last->ee_block, shift);
                                        ext4_ext_try_to_merge_right(inode, path,
                                                ex_last);
                                        ex_last--;
                                }
                        }
                        err = ext4_ext_dirty(handle, inode, path + depth);
                        if (err)
                                goto out;

                        if (--depth < 0 || !update)
                                break;
                }

                /* Update index too */
                err = ext4_ext_get_access(handle, inode, path + depth);
                if (err)
                        goto out;

                if (SHIFT == SHIFT_LEFT)
                        le32_add_cpu(&path[depth].p_idx->ei_block, -shift);
                else
                        le32_add_cpu(&path[depth].p_idx->ei_block, shift);
                err = ext4_ext_dirty(handle, inode, path + depth);
                if (err)
                        goto out;

                /* we are done if current index is not a starting index */
                if (path[depth].p_idx != EXT_FIRST_INDEX(path[depth].p_hdr))
                        break;

                depth--;
        }

out:
        return err;
}

/*
 * ext4_ext_shift_extents:
 * All the extents which lies in the range from @start to the last allocated
 * block for the @inode are shifted either towards left or right (depending
 * upon @SHIFT) by @shift blocks.
 * On success, 0 is returned, error otherwise.
 */
static int
ext4_ext_shift_extents(struct inode *inode, handle_t *handle,
                       ext4_lblk_t start, ext4_lblk_t shift,
                       enum SHIFT_DIRECTION SHIFT)
{
        struct ext4_ext_path *path;
        int ret = 0, depth;
        struct ext4_extent *extent;
        ext4_lblk_t stop, *iterator, ex_start, ex_end;
        ext4_lblk_t tmp = EXT_MAX_BLOCKS;

        /* Let path point to the last extent */
        path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
                                EXT4_EX_NOCACHE);
        if (IS_ERR(path))
                return PTR_ERR(path);

        depth = path->p_depth;
        extent = path[depth].p_ext;
        if (!extent)
                goto out;

        stop = le32_to_cpu(extent->ee_block);

       /*
        * For left shifts, make sure the hole on the left is big enough to
        * accommodate the shift.  For right shifts, make sure the last extent
        * won't be shifted beyond EXT_MAX_BLOCKS.
        */
        if (SHIFT == SHIFT_LEFT) {
                path = ext4_find_extent(inode, start - 1, path,
                                        EXT4_EX_NOCACHE);
                if (IS_ERR(path))
                        return PTR_ERR(path);
                depth = path->p_depth;
                extent =  path[depth].p_ext;
                if (extent) {
                        ex_start = le32_to_cpu(extent->ee_block);
                        ex_end = le32_to_cpu(extent->ee_block) +
                                ext4_ext_get_actual_len(extent);
                } else {
                        ex_start = 0;
                        ex_end = 0;
                }

                if ((start == ex_start && shift > ex_start) ||
                    (shift > start - ex_end)) {
                        ret = -EINVAL;
                        goto out;
                }
        } else {
                if (shift > EXT_MAX_BLOCKS -
                    (stop + ext4_ext_get_actual_len(extent))) {
                        ret = -EINVAL;
                        goto out;
                }
        }

        /*
         * In case of left shift, iterator points to start and it is increased
         * till we reach stop. In case of right shift, iterator points to stop
         * and it is decreased till we reach start.
         */
again:
        ret = 0;
        if (SHIFT == SHIFT_LEFT)
                iterator = &start;
        else
                iterator = &stop;

        if (tmp != EXT_MAX_BLOCKS)
                *iterator = tmp;

        /*
         * Its safe to start updating extents.  Start and stop are unsigned, so
         * in case of right shift if extent with 0 block is reached, iterator
         * becomes NULL to indicate the end of the loop.
         */
        while (iterator && start <= stop) {
                path = ext4_find_extent(inode, *iterator, path,
                                        EXT4_EX_NOCACHE);
                if (IS_ERR(path))
                        return PTR_ERR(path);
                depth = path->p_depth;
                extent = path[depth].p_ext;
                if (!extent) {
                        EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
                                         (unsigned long) *iterator);
                        ret = -EFSCORRUPTED;
                        goto out;
                }
                if (SHIFT == SHIFT_LEFT && *iterator >
                    le32_to_cpu(extent->ee_block)) {
                        /* Hole, move to the next extent */
                        if (extent < EXT_LAST_EXTENT(path[depth].p_hdr)) {
                                path[depth].p_ext++;
                        } else {
                                *iterator = ext4_ext_next_allocated_block(path);
                                continue;
                        }
                }

                tmp = *iterator;
                if (SHIFT == SHIFT_LEFT) {
                        extent = EXT_LAST_EXTENT(path[depth].p_hdr);
                        *iterator = le32_to_cpu(extent->ee_block) +
                                        ext4_ext_get_actual_len(extent);
                } else {
                        extent = EXT_FIRST_EXTENT(path[depth].p_hdr);
                        if (le32_to_cpu(extent->ee_block) > start)
                                *iterator = le32_to_cpu(extent->ee_block) - 1;
                        else if (le32_to_cpu(extent->ee_block) == start)
                                iterator = NULL;
                        else {
                                extent = EXT_LAST_EXTENT(path[depth].p_hdr);
                                while (le32_to_cpu(extent->ee_block) >= start)
                                        extent--;

                                if (extent == EXT_LAST_EXTENT(path[depth].p_hdr))
                                        break;

                                extent++;
                                iterator = NULL;
                        }
                        path[depth].p_ext = extent;
                }
                ret = ext4_ext_shift_path_extents(path, shift, inode,
                                handle, SHIFT);
                /* iterator can be NULL which means we should break */
                if (ret == -EAGAIN)
                        goto again;
                if (ret)
                        break;
        }
out:
        ext4_free_ext_path(path);
        return ret;
}

/*
 * ext4_collapse_range:
 * This implements the fallocate's collapse range functionality for ext4
 * Returns: 0 and non-zero on error.
 */
static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len)
{
        struct inode *inode = file_inode(file);
        struct super_block *sb = inode->i_sb;
        struct address_space *mapping = inode->i_mapping;
        loff_t end = offset + len;
        ext4_lblk_t start_lblk, end_lblk;
        handle_t *handle;
        unsigned int credits;
        loff_t start, new_size;
        int ret;

        trace_ext4_collapse_range(inode, offset, len);
        WARN_ON_ONCE(!inode_is_locked(inode));

        /* Currently just for extent based files */
        if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
                return -EOPNOTSUPP;
        /* Collapse range works only on fs cluster size aligned regions. */
        if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb)))
                return -EINVAL;
        /*
         * There is no need to overlap collapse range with EOF, in which case
         * it is effectively a truncate operation
         */
        if (end >= inode->i_size)
                return -EINVAL;

        /*
         * Write tail of the last page before removed range and data that
         * will be shifted since they will get removed from the page cache
         * below. We are also protected from pages becoming dirty by
         * i_rwsem and invalidate_lock.
         * Need to round down offset to be aligned with page size boundary
         * for page size > block size.
         */
        start = round_down(offset, PAGE_SIZE);
        ret = filemap_write_and_wait_range(mapping, start, offset);
        if (!ret)
                ret = filemap_write_and_wait_range(mapping, end, LLONG_MAX);
        if (ret)
                return ret;

        truncate_pagecache(inode, start);

        credits = ext4_chunk_trans_extent(inode, 0);
        handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
        if (IS_ERR(handle))
                return PTR_ERR(handle);

        ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_FALLOC_RANGE, handle);

        start_lblk = offset >> inode->i_blkbits;
        end_lblk = (offset + len) >> inode->i_blkbits;

        ext4_check_map_extents_env(inode);

        down_write(&EXT4_I(inode)->i_data_sem);
        ext4_discard_preallocations(inode);
        ext4_es_remove_extent(inode, start_lblk, EXT_MAX_BLOCKS - start_lblk);

        ret = ext4_ext_remove_space(inode, start_lblk, end_lblk - 1);
        if (ret) {
                up_write(&EXT4_I(inode)->i_data_sem);
                goto out_handle;
        }
        ext4_discard_preallocations(inode);

        ret = ext4_ext_shift_extents(inode, handle, end_lblk,
                                     end_lblk - start_lblk, SHIFT_LEFT);
        if (ret) {
                up_write(&EXT4_I(inode)->i_data_sem);
                goto out_handle;
        }

        new_size = inode->i_size - len;
        i_size_write(inode, new_size);
        EXT4_I(inode)->i_disksize = new_size;

        up_write(&EXT4_I(inode)->i_data_sem);
        ret = ext4_mark_inode_dirty(handle, inode);
        if (ret)
                goto out_handle;

        ext4_update_inode_fsync_trans(handle, inode, 1);
        if ((file->f_flags & O_SYNC) || IS_SYNC(inode))
                ext4_handle_sync(handle);

out_handle:
        ext4_journal_stop(handle);
        return ret;
}

/*
 * ext4_insert_range:
 * This function implements the FALLOC_FL_INSERT_RANGE flag of fallocate.
 * The data blocks starting from @offset to the EOF are shifted by @len
 * towards right to create a hole in the @inode. Inode size is increased
 * by len bytes.
 * Returns 0 on success, error otherwise.
 */
static int ext4_insert_range(struct file *file, loff_t offset, loff_t len)
{
        struct inode *inode = file_inode(file);
        struct super_block *sb = inode->i_sb;
        struct address_space *mapping = inode->i_mapping;
        handle_t *handle;
        struct ext4_ext_path *path;
        struct ext4_extent *extent;
        ext4_lblk_t start_lblk, len_lblk, ee_start_lblk = 0;
        unsigned int credits, ee_len;
        int ret, depth;
        loff_t start;

        trace_ext4_insert_range(inode, offset, len);
        WARN_ON_ONCE(!inode_is_locked(inode));

        /* Currently just for extent based files */
        if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
                return -EOPNOTSUPP;
        /* Insert range works only on fs cluster size aligned regions. */
        if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb)))
                return -EINVAL;
        /* Offset must be less than i_size */
        if (offset >= inode->i_size)
                return -EINVAL;
        /* Check whether the maximum file size would be exceeded */
        if (len > inode->i_sb->s_maxbytes - inode->i_size)
                return -EFBIG;

        /*
         * Write out all dirty pages. Need to round down to align start offset
         * to page size boundary for page size > block size.
         */
        start = round_down(offset, PAGE_SIZE);
        ret = filemap_write_and_wait_range(mapping, start, LLONG_MAX);
        if (ret)
                return ret;

        truncate_pagecache(inode, start);

        credits = ext4_chunk_trans_extent(inode, 0);
        handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
        if (IS_ERR(handle))
                return PTR_ERR(handle);

        ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_FALLOC_RANGE, handle);

        /* Expand file to avoid data loss if there is error while shifting */
        inode->i_size += len;
        EXT4_I(inode)->i_disksize += len;
        ret = ext4_mark_inode_dirty(handle, inode);
        if (ret)
                goto out_handle;

        start_lblk = offset >> inode->i_blkbits;
        len_lblk = len >> inode->i_blkbits;

        ext4_check_map_extents_env(inode);

        down_write(&EXT4_I(inode)->i_data_sem);
        ext4_discard_preallocations(inode);

        path = ext4_find_extent(inode, start_lblk, NULL, 0);
        if (IS_ERR(path)) {
                up_write(&EXT4_I(inode)->i_data_sem);
                ret = PTR_ERR(path);
                goto out_handle;
        }

        depth = ext_depth(inode);
        extent = path[depth].p_ext;
        if (extent) {
                ee_start_lblk = le32_to_cpu(extent->ee_block);
                ee_len = ext4_ext_get_actual_len(extent);

                /*
                 * If start_lblk is not the starting block of extent, split
                 * the extent @start_lblk
                 */
                if ((start_lblk > ee_start_lblk) &&
                                (start_lblk < (ee_start_lblk + ee_len))) {
                        path = ext4_split_extent_at(handle, inode, path,
                                        start_lblk, EXT4_EX_NOCACHE |
                                        EXT4_GET_BLOCKS_SPLIT_NOMERGE |
                                        EXT4_GET_BLOCKS_METADATA_NOFAIL);
                }

                if (IS_ERR(path)) {
                        up_write(&EXT4_I(inode)->i_data_sem);
                        ret = PTR_ERR(path);
                        goto out_handle;
                }
        }

        ext4_free_ext_path(path);
        ext4_es_remove_extent(inode, start_lblk, EXT_MAX_BLOCKS - start_lblk);

        /*
         * if start_lblk lies in a hole which is at start of file, use
         * ee_start_lblk to shift extents
         */
        ret = ext4_ext_shift_extents(inode, handle,
                max(ee_start_lblk, start_lblk), len_lblk, SHIFT_RIGHT);
        up_write(&EXT4_I(inode)->i_data_sem);
        if (ret)
                goto out_handle;

        ext4_update_inode_fsync_trans(handle, inode, 1);
        if ((file->f_flags & O_SYNC) || IS_SYNC(inode))
                ext4_handle_sync(handle);

out_handle:
        ext4_journal_stop(handle);
        return ret;
}

/**
 * ext4_swap_extents() - Swap extents between two inodes
 * @handle: handle for this transaction
 * @inode1:        First inode
 * @inode2:        Second inode
 * @lblk1:        Start block for first inode
 * @lblk2:        Start block for second inode
 * @count:        Number of blocks to swap
 * @unwritten: Mark second inode's extents as unwritten after swap
 * @erp:        Pointer to save error value
 *
 * This helper routine does exactly what is promise "swap extents". All other
 * stuff such as page-cache locking consistency, bh mapping consistency or
 * extent's data copying must be performed by caller.
 * Locking:
 *                i_rwsem is held for both inodes
 *                 i_data_sem is locked for write for both inodes
 * Assumptions:
 *                All pages from requested range are locked for both inodes
 */
int
ext4_swap_extents(handle_t *handle, struct inode *inode1,
                  struct inode *inode2, ext4_lblk_t lblk1, ext4_lblk_t lblk2,
                  ext4_lblk_t count, int unwritten, int *erp)
{
        struct ext4_ext_path *path1 = NULL;
        struct ext4_ext_path *path2 = NULL;
        int replaced_count = 0;

        BUG_ON(!rwsem_is_locked(&EXT4_I(inode1)->i_data_sem));
        BUG_ON(!rwsem_is_locked(&EXT4_I(inode2)->i_data_sem));
        BUG_ON(!inode_is_locked(inode1));
        BUG_ON(!inode_is_locked(inode2));

        ext4_es_remove_extent(inode1, lblk1, count);
        ext4_es_remove_extent(inode2, lblk2, count);

        while (count) {
                struct ext4_extent *ex1, *ex2, tmp_ex;
                ext4_lblk_t e1_blk, e2_blk;
                int e1_len, e2_len, len;
                int split = 0;

                path1 = ext4_find_extent(inode1, lblk1, path1, EXT4_EX_NOCACHE);
                if (IS_ERR(path1)) {
                        *erp = PTR_ERR(path1);
                        goto errout;
                }
                path2 = ext4_find_extent(inode2, lblk2, path2, EXT4_EX_NOCACHE);
                if (IS_ERR(path2)) {
                        *erp = PTR_ERR(path2);
                        goto errout;
                }
                ex1 = path1[path1->p_depth].p_ext;
                ex2 = path2[path2->p_depth].p_ext;
                /* Do we have something to swap ? */
                if (unlikely(!ex2 || !ex1))
                        goto errout;

                e1_blk = le32_to_cpu(ex1->ee_block);
                e2_blk = le32_to_cpu(ex2->ee_block);
                e1_len = ext4_ext_get_actual_len(ex1);
                e2_len = ext4_ext_get_actual_len(ex2);

                /* Hole handling */
                if (!in_range(lblk1, e1_blk, e1_len) ||
                    !in_range(lblk2, e2_blk, e2_len)) {
                        ext4_lblk_t next1, next2;

                        /* if hole after extent, then go to next extent */
                        next1 = ext4_ext_next_allocated_block(path1);
                        next2 = ext4_ext_next_allocated_block(path2);
                        /* If hole before extent, then shift to that extent */
                        if (e1_blk > lblk1)
                                next1 = e1_blk;
                        if (e2_blk > lblk2)
                                next2 = e2_blk;
                        /* Do we have something to swap */
                        if (next1 == EXT_MAX_BLOCKS || next2 == EXT_MAX_BLOCKS)
                                goto errout;
                        /* Move to the rightest boundary */
                        len = next1 - lblk1;
                        if (len < next2 - lblk2)
                                len = next2 - lblk2;
                        if (len > count)
                                len = count;
                        lblk1 += len;
                        lblk2 += len;
                        count -= len;
                        continue;
                }

                /* Prepare left boundary */
                if (e1_blk < lblk1) {
                        split = 1;
                        path1 = ext4_force_split_extent_at(handle, inode1,
                                                           path1, lblk1, 0);
                        if (IS_ERR(path1)) {
                                *erp = PTR_ERR(path1);
                                goto errout;
                        }
                }
                if (e2_blk < lblk2) {
                        split = 1;
                        path2 = ext4_force_split_extent_at(handle, inode2,
                                                           path2, lblk2, 0);
                        if (IS_ERR(path2)) {
                                *erp = PTR_ERR(path2);
                                goto errout;
                        }
                }
                /* ext4_split_extent_at() may result in leaf extent split,
                 * path must to be revalidated. */
                if (split)
                        continue;

                /* Prepare right boundary */
                len = count;
                if (len > e1_blk + e1_len - lblk1)
                        len = e1_blk + e1_len - lblk1;
                if (len > e2_blk + e2_len - lblk2)
                        len = e2_blk + e2_len - lblk2;

                if (len != e1_len) {
                        split = 1;
                        path1 = ext4_force_split_extent_at(handle, inode1,
                                                        path1, lblk1 + len, 0);
                        if (IS_ERR(path1)) {
                                *erp = PTR_ERR(path1);
                                goto errout;
                        }
                }
                if (len != e2_len) {
                        split = 1;
                        path2 = ext4_force_split_extent_at(handle, inode2,
                                                        path2, lblk2 + len, 0);
                        if (IS_ERR(path2)) {
                                *erp = PTR_ERR(path2);
                                goto errout;
                        }
                }
                /* ext4_split_extent_at() may result in leaf extent split,
                 * path must to be revalidated. */
                if (split)
                        continue;

                BUG_ON(e2_len != e1_len);
                *erp = ext4_ext_get_access(handle, inode1, path1 + path1->p_depth);
                if (unlikely(*erp))
                        goto errout;
                *erp = ext4_ext_get_access(handle, inode2, path2 + path2->p_depth);
                if (unlikely(*erp))
                        goto errout;

                /* Both extents are fully inside boundaries. Swap it now */
                tmp_ex = *ex1;
                ext4_ext_store_pblock(ex1, ext4_ext_pblock(ex2));
                ext4_ext_store_pblock(ex2, ext4_ext_pblock(&tmp_ex));
                ex1->ee_len = cpu_to_le16(e2_len);
                ex2->ee_len = cpu_to_le16(e1_len);
                if (unwritten)
                        ext4_ext_mark_unwritten(ex2);
                if (ext4_ext_is_unwritten(&tmp_ex))
                        ext4_ext_mark_unwritten(ex1);

                ext4_ext_try_to_merge(handle, inode2, path2, ex2);
                ext4_ext_try_to_merge(handle, inode1, path1, ex1);
                *erp = ext4_ext_dirty(handle, inode2, path2 +
                                      path2->p_depth);
                if (unlikely(*erp))
                        goto errout;
                *erp = ext4_ext_dirty(handle, inode1, path1 +
                                      path1->p_depth);
                /*
                 * Looks scarry ah..? second inode already points to new blocks,
                 * and it was successfully dirtied. But luckily error may happen
                 * only due to journal error, so full transaction will be
                 * aborted anyway.
                 */
                if (unlikely(*erp))
                        goto errout;

                lblk1 += len;
                lblk2 += len;
                replaced_count += len;
                count -= len;
        }

errout:
        ext4_free_ext_path(path1);
        ext4_free_ext_path(path2);
        return replaced_count;
}

/*
 * ext4_clu_mapped - determine whether any block in a logical cluster has
 *                   been mapped to a physical cluster
 *
 * @inode - file containing the logical cluster
 * @lclu - logical cluster of interest
 *
 * Returns 1 if any block in the logical cluster is mapped, signifying
 * that a physical cluster has been allocated for it.  Otherwise,
 * returns 0.  Can also return negative error codes.  Derived from
 * ext4_ext_map_blocks().
 */
int ext4_clu_mapped(struct inode *inode, ext4_lblk_t lclu)
{
        struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
        struct ext4_ext_path *path;
        int depth, mapped = 0, err = 0;
        struct ext4_extent *extent;
        ext4_lblk_t first_lblk, first_lclu, last_lclu;

        /*
         * if data can be stored inline, the logical cluster isn't
         * mapped - no physical clusters have been allocated, and the
         * file has no extents
         */
        if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) ||
            ext4_has_inline_data(inode))
                return 0;

        /* search for the extent closest to the first block in the cluster */
        path = ext4_find_extent(inode, EXT4_C2B(sbi, lclu), NULL, 0);
        if (IS_ERR(path))
                return PTR_ERR(path);

        depth = ext_depth(inode);

        /*
         * A consistent leaf must not be empty.  This situation is possible,
         * though, _during_ tree modification, and it's why an assert can't
         * be put in ext4_find_extent().
         */
        if (unlikely(path[depth].p_ext == NULL && depth != 0)) {
                EXT4_ERROR_INODE(inode,
                    "bad extent address - lblock: %lu, depth: %d, pblock: %lld",
                                 (unsigned long) EXT4_C2B(sbi, lclu),
                                 depth, path[depth].p_block);
                err = -EFSCORRUPTED;
                goto out;
        }

        extent = path[depth].p_ext;

        /* can't be mapped if the extent tree is empty */
        if (extent == NULL)
                goto out;

        first_lblk = le32_to_cpu(extent->ee_block);
        first_lclu = EXT4_B2C(sbi, first_lblk);

        /*
         * Three possible outcomes at this point - found extent spanning
         * the target cluster, to the left of the target cluster, or to the
         * right of the target cluster.  The first two cases are handled here.
         * The last case indicates the target cluster is not mapped.
         */
        if (lclu >= first_lclu) {
                last_lclu = EXT4_B2C(sbi, first_lblk +
                                     ext4_ext_get_actual_len(extent) - 1);
                if (lclu <= last_lclu) {
                        mapped = 1;
                } else {
                        first_lblk = ext4_ext_next_allocated_block(path);
                        first_lclu = EXT4_B2C(sbi, first_lblk);
                        if (lclu == first_lclu)
                                mapped = 1;
                }
        }

out:
        ext4_free_ext_path(path);

        return err ? err : mapped;
}

/*
 * Updates physical block address and unwritten status of extent
 * starting at lblk start and of len. If such an extent doesn't exist,
 * this function splits the extent tree appropriately to create an
 * extent like this.  This function is called in the fast commit
 * replay path.  Returns 0 on success and error on failure.
 */
int ext4_ext_replay_update_ex(struct inode *inode, ext4_lblk_t start,
                              int len, int unwritten, ext4_fsblk_t pblk)
{
        struct ext4_ext_path *path;
        struct ext4_extent *ex;
        int ret;

        path = ext4_find_extent(inode, start, NULL, 0);
        if (IS_ERR(path))
                return PTR_ERR(path);
        ex = path[path->p_depth].p_ext;
        if (!ex) {
                ret = -EFSCORRUPTED;
                goto out;
        }

        if (le32_to_cpu(ex->ee_block) != start ||
                ext4_ext_get_actual_len(ex) != len) {
                /* We need to split this extent to match our extent first */
                down_write(&EXT4_I(inode)->i_data_sem);
                path = ext4_force_split_extent_at(NULL, inode, path, start, 1);
                up_write(&EXT4_I(inode)->i_data_sem);
                if (IS_ERR(path)) {
                        ret = PTR_ERR(path);
                        goto out;
                }

                path = ext4_find_extent(inode, start, path, 0);
                if (IS_ERR(path))
                        return PTR_ERR(path);

                ex = path[path->p_depth].p_ext;
                WARN_ON(le32_to_cpu(ex->ee_block) != start);

                if (ext4_ext_get_actual_len(ex) != len) {
                        down_write(&EXT4_I(inode)->i_data_sem);
                        path = ext4_force_split_extent_at(NULL, inode, path,
                                                          start + len, 1);
                        up_write(&EXT4_I(inode)->i_data_sem);
                        if (IS_ERR(path)) {
                                ret = PTR_ERR(path);
                                goto out;
                        }

                        path = ext4_find_extent(inode, start, path, 0);
                        if (IS_ERR(path))
                                return PTR_ERR(path);
                        ex = path[path->p_depth].p_ext;
                }
        }
        if (unwritten)
                ext4_ext_mark_unwritten(ex);
        else
                ext4_ext_mark_initialized(ex);
        ext4_ext_store_pblock(ex, pblk);
        down_write(&EXT4_I(inode)->i_data_sem);
        ret = ext4_ext_dirty(NULL, inode, &path[path->p_depth]);
        up_write(&EXT4_I(inode)->i_data_sem);
out:
        ext4_free_ext_path(path);
        ext4_mark_inode_dirty(NULL, inode);
        return ret;
}

/* Try to shrink the extent tree */
void ext4_ext_replay_shrink_inode(struct inode *inode, ext4_lblk_t end)
{
        struct ext4_ext_path *path = NULL;
        struct ext4_extent *ex;
        ext4_lblk_t old_cur, cur = 0;

        while (cur < end) {
                path = ext4_find_extent(inode, cur, NULL, 0);
                if (IS_ERR(path))
                        return;
                ex = path[path->p_depth].p_ext;
                if (!ex) {
                        ext4_free_ext_path(path);
                        ext4_mark_inode_dirty(NULL, inode);
                        return;
                }
                old_cur = cur;
                cur = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex);
                if (cur <= old_cur)
                        cur = old_cur + 1;
                ext4_ext_try_to_merge(NULL, inode, path, ex);
                down_write(&EXT4_I(inode)->i_data_sem);
                ext4_ext_dirty(NULL, inode, &path[path->p_depth]);
                up_write(&EXT4_I(inode)->i_data_sem);
                ext4_mark_inode_dirty(NULL, inode);
                ext4_free_ext_path(path);
        }
}

/* Check if *cur is a hole and if it is, skip it */
static int skip_hole(struct inode *inode, ext4_lblk_t *cur)
{
        int ret;
        struct ext4_map_blocks map;

        map.m_lblk = *cur;
        map.m_len = ((inode->i_size) >> inode->i_sb->s_blocksize_bits) - *cur;

        ret = ext4_map_blocks(NULL, inode, &map, 0);
        if (ret < 0)
                return ret;
        if (ret != 0)
                return 0;
        *cur = *cur + map.m_len;
        return 0;
}

/* Count number of blocks used by this inode and update i_blocks */
int ext4_ext_replay_set_iblocks(struct inode *inode)
{
        struct ext4_ext_path *path = NULL, *path2 = NULL;
        struct ext4_extent *ex;
        ext4_lblk_t cur = 0, end;
        int numblks = 0, i, ret = 0;
        ext4_fsblk_t cmp1, cmp2;
        struct ext4_map_blocks map;

        /* Determin the size of the file first */
        path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
                                        EXT4_EX_NOCACHE);
        if (IS_ERR(path))
                return PTR_ERR(path);
        ex = path[path->p_depth].p_ext;
        if (!ex)
                goto out;
        end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex);

        /* Count the number of data blocks */
        cur = 0;
        while (cur < end) {
                map.m_lblk = cur;
                map.m_len = end - cur;
                ret = ext4_map_blocks(NULL, inode, &map, 0);
                if (ret < 0)
                        break;
                if (ret > 0)
                        numblks += ret;
                cur = cur + map.m_len;
        }

        /*
         * Count the number of extent tree blocks. We do it by looking up
         * two successive extents and determining the difference between
         * their paths. When path is different for 2 successive extents
         * we compare the blocks in the path at each level and increment
         * iblocks by total number of differences found.
         */
        cur = 0;
        ret = skip_hole(inode, &cur);
        if (ret < 0)
                goto out;
        path = ext4_find_extent(inode, cur, path, 0);
        if (IS_ERR(path))
                goto out;
        numblks += path->p_depth;
        while (cur < end) {
                path = ext4_find_extent(inode, cur, path, 0);
                if (IS_ERR(path))
                        break;
                ex = path[path->p_depth].p_ext;
                if (!ex)
                        goto cleanup;

                cur = max(cur + 1, le32_to_cpu(ex->ee_block) +
                                        ext4_ext_get_actual_len(ex));
                ret = skip_hole(inode, &cur);
                if (ret < 0)
                        break;

                path2 = ext4_find_extent(inode, cur, path2, 0);
                if (IS_ERR(path2))
                        break;

                for (i = 0; i <= max(path->p_depth, path2->p_depth); i++) {
                        cmp1 = cmp2 = 0;
                        if (i <= path->p_depth)
                                cmp1 = path[i].p_bh ?
                                        path[i].p_bh->b_blocknr : 0;
                        if (i <= path2->p_depth)
                                cmp2 = path2[i].p_bh ?
                                        path2[i].p_bh->b_blocknr : 0;
                        if (cmp1 != cmp2 && cmp2 != 0)
                                numblks++;
                }
        }

out:
        inode->i_blocks = numblks << (inode->i_sb->s_blocksize_bits - 9);
        ext4_mark_inode_dirty(NULL, inode);
cleanup:
        ext4_free_ext_path(path);
        ext4_free_ext_path(path2);
        return 0;
}

int ext4_ext_clear_bb(struct inode *inode)
{
        struct ext4_ext_path *path = NULL;
        struct ext4_extent *ex;
        ext4_lblk_t cur = 0, end;
        int j, ret = 0;
        struct ext4_map_blocks map;

        if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA))
                return 0;

        /* Determin the size of the file first */
        path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
                                        EXT4_EX_NOCACHE);
        if (IS_ERR(path))
                return PTR_ERR(path);
        ex = path[path->p_depth].p_ext;
        if (!ex)
                goto out;
        end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex);

        cur = 0;
        while (cur < end) {
                map.m_lblk = cur;
                map.m_len = end - cur;
                ret = ext4_map_blocks(NULL, inode, &map, 0);
                if (ret < 0)
                        break;
                if (ret > 0) {
                        path = ext4_find_extent(inode, map.m_lblk, path, 0);
                        if (!IS_ERR(path)) {
                                for (j = 0; j < path->p_depth; j++) {
                                        ext4_mb_mark_bb(inode->i_sb,
                                                        path[j].p_block, 1, false);
                                        ext4_fc_record_regions(inode->i_sb, inode->i_ino,
                                                        0, path[j].p_block, 1, 1);
                                }
                        } else {
                                path = NULL;
                        }
                        ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false);
                        ext4_fc_record_regions(inode->i_sb, inode->i_ino,
                                        map.m_lblk, map.m_pblk, map.m_len, 1);
                }
                cur = cur + map.m_len;
        }

out:
        ext4_free_ext_path(path);
        return 0;
}

#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS)
int ext4_ext_space_root_idx_test(struct inode *inode, int check)
{
        return ext4_ext_space_root_idx(inode, check);
}
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_space_root_idx_test);

struct ext4_ext_path *ext4_split_convert_extents_test(handle_t *handle,
                        struct inode *inode, struct ext4_map_blocks *map,
                        struct ext4_ext_path *path, int flags,
                        unsigned int *allocated)
{
        return ext4_split_convert_extents(handle, inode, map, path,
                                          flags, allocated);
}
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_split_convert_extents_test);

EXPORT_SYMBOL_FOR_EXT4_TEST(__ext4_ext_dirty);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_zeroout);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_register_shrinker);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_unregister_shrinker);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_map_create_blocks);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_init_tree);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_lookup_extent);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_insert_extent);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_insert_extent);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_find_extent);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_issue_zeroout);
EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_map_query_blocks);
#endif













































































































































































































































































































































































































































































































































































































































   42 













































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   81 







   81 




























































   81 









































































































































































































































































































































































































































































































  100 
   77 
   89 










































































































































































  101 














  100 











































   22 














   22 



















    1 





    1 


















































    1 




































































































































   10 




   10 














































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
// SPDX-License-Identifier: GPL-2.0-only
/*
 * mm/page-writeback.c
 *
 * Copyright (C) 2002, Linus Torvalds.
 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
 *
 * Contains functions related to writing back dirty pages at the
 * address_space level.
 *
 * 10Apr2002        Andrew Morton
 *                Initial version
 */

#include <linux/kernel.h>
#include <linux/math64.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/pagemap.h>
#include <linux/writeback.h>
#include <linux/init.h>
#include <linux/backing-dev.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/blkdev.h>
#include <linux/mpage.h>
#include <linux/rmap.h>
#include <linux/percpu.h>
#include <linux/smp.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/syscalls.h>
#include <linux/folio_batch.h>
#include <linux/timer.h>
#include <linux/sched/rt.h>
#include <linux/sched/signal.h>
#include <linux/mm_inline.h>
#include <linux/shmem_fs.h>
#include <trace/events/writeback.h>

#include "internal.h"

/*
 * Sleep at most 200ms at a time in balance_dirty_pages().
 */
#define MAX_PAUSE                max(HZ/5, 1)

/*
 * Try to keep balance_dirty_pages() call intervals higher than this many pages
 * by raising pause time to max_pause when falls below it.
 */
#define DIRTY_POLL_THRESH        (128 >> (PAGE_SHIFT - 10))

/*
 * Estimate write bandwidth or update dirty limit at 200ms intervals.
 */
#define BANDWIDTH_INTERVAL        max(HZ/5, 1)

#define RATELIMIT_CALC_SHIFT        10

/*
 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
 * will look to see if it needs to force writeback or throttling.
 */
static long ratelimit_pages = 32;

/* The following parameters are exported via /proc/sys/vm */

/*
 * Start background writeback (via writeback threads) at this percentage
 */
static int dirty_background_ratio = 10;

/*
 * dirty_background_bytes starts at 0 (disabled) so that it is a function of
 * dirty_background_ratio * the amount of dirtyable memory
 */
static unsigned long dirty_background_bytes;

/*
 * free highmem will not be subtracted from the total free memory
 * for calculating free ratios if vm_highmem_is_dirtyable is true
 */
static int vm_highmem_is_dirtyable;

/*
 * The generator of dirty data starts writeback at this percentage
 */
static int vm_dirty_ratio = 20;

/*
 * vm_dirty_bytes starts at 0 (disabled) so that it is a function of
 * vm_dirty_ratio * the amount of dirtyable memory
 */
static unsigned long vm_dirty_bytes;

/*
 * The interval between `kupdate'-style writebacks
 */
unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */

EXPORT_SYMBOL_GPL(dirty_writeback_interval);

/*
 * The longest time for which data is allowed to remain dirty
 */
unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */

/* End of sysctl-exported parameters */

struct wb_domain global_wb_domain;

/*
 * Length of period for aging writeout fractions of bdis. This is an
 * arbitrarily chosen number. The longer the period, the slower fractions will
 * reflect changes in current writeout rate.
 */
#define VM_COMPLETIONS_PERIOD_LEN (3*HZ)

#ifdef CONFIG_CGROUP_WRITEBACK

#define GDTC_INIT(__wb)                .wb = (__wb),                                \
                                .dom = &global_wb_domain,                \
                                .wb_completions = &(__wb)->completions

#define GDTC_INIT_NO_WB                .dom = &global_wb_domain

#define MDTC_INIT(__wb, __gdtc)        .wb = (__wb),                                \
                                .dom = mem_cgroup_wb_domain(__wb),        \
                                .wb_completions = &(__wb)->memcg_completions, \
                                .gdtc = __gdtc

static bool mdtc_valid(struct dirty_throttle_control *dtc)
{
        return dtc->dom;
}

static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
{
        return dtc->dom;
}

static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
{
        return mdtc->gdtc;
}

static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
{
        return &wb->memcg_completions;
}

static void wb_min_max_ratio(struct bdi_writeback *wb,
                             unsigned long *minp, unsigned long *maxp)
{
        unsigned long this_bw = READ_ONCE(wb->avg_write_bandwidth);
        unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
        unsigned long long min = wb->bdi->min_ratio;
        unsigned long long max = wb->bdi->max_ratio;

        /*
         * @wb may already be clean by the time control reaches here and
         * the total may not include its bw.
         */
        if (this_bw < tot_bw) {
                if (min) {
                        min *= this_bw;
                        min = div64_ul(min, tot_bw);
                }
                if (max < 100 * BDI_RATIO_SCALE) {
                        max *= this_bw;
                        max = div64_ul(max, tot_bw);
                }
        }

        *minp = min;
        *maxp = max;
}

#else        /* CONFIG_CGROUP_WRITEBACK */

#define GDTC_INIT(__wb)                .wb = (__wb),                           \
                                .wb_completions = &(__wb)->completions
#define GDTC_INIT_NO_WB
#define MDTC_INIT(__wb, __gdtc)

static bool mdtc_valid(struct dirty_throttle_control *dtc)
{
        return false;
}

static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
{
        return &global_wb_domain;
}

static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
{
        return NULL;
}

static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
{
        return NULL;
}

static void wb_min_max_ratio(struct bdi_writeback *wb,
                             unsigned long *minp, unsigned long *maxp)
{
        *minp = wb->bdi->min_ratio;
        *maxp = wb->bdi->max_ratio;
}

#endif        /* CONFIG_CGROUP_WRITEBACK */

/*
 * In a memory zone, there is a certain amount of pages we consider
 * available for the page cache, which is essentially the number of
 * free and reclaimable pages, minus some zone reserves to protect
 * lowmem and the ability to uphold the zone's watermarks without
 * requiring writeback.
 *
 * This number of dirtyable pages is the base value of which the
 * user-configurable dirty ratio is the effective number of pages that
 * are allowed to be actually dirtied.  Per individual zone, or
 * globally by using the sum of dirtyable pages over all zones.
 *
 * Because the user is allowed to specify the dirty limit globally as
 * absolute number of bytes, calculating the per-zone dirty limit can
 * require translating the configured limit into a percentage of
 * global dirtyable memory first.
 */

/**
 * node_dirtyable_memory - number of dirtyable pages in a node
 * @pgdat: the node
 *
 * Return: the node's number of pages potentially available for dirty
 * page cache.  This is the base value for the per-node dirty limits.
 */
static unsigned long node_dirtyable_memory(struct pglist_data *pgdat)
{
        unsigned long nr_pages = 0;
        int z;

        for (z = 0; z < MAX_NR_ZONES; z++) {
                struct zone *zone = pgdat->node_zones + z;

                if (!populated_zone(zone))
                        continue;

                nr_pages += zone_page_state(zone, NR_FREE_PAGES);
        }

        /*
         * Pages reserved for the kernel should not be considered
         * dirtyable, to prevent a situation where reclaim has to
         * clean pages in order to balance the zones.
         */
        nr_pages -= min(nr_pages, pgdat->totalreserve_pages);

        nr_pages += node_page_state(pgdat, NR_INACTIVE_FILE);
        nr_pages += node_page_state(pgdat, NR_ACTIVE_FILE);

        return nr_pages;
}

static unsigned long highmem_dirtyable_memory(unsigned long total)
{
#ifdef CONFIG_HIGHMEM
        int node;
        unsigned long x = 0;
        int i;

        for_each_node_state(node, N_HIGH_MEMORY) {
                for (i = ZONE_NORMAL + 1; i < MAX_NR_ZONES; i++) {
                        struct zone *z;
                        unsigned long nr_pages;

                        if (!is_highmem_idx(i))
                                continue;

                        z = &NODE_DATA(node)->node_zones[i];
                        if (!populated_zone(z))
                                continue;

                        nr_pages = zone_page_state(z, NR_FREE_PAGES);
                        /* watch for underflows */
                        nr_pages -= min(nr_pages, high_wmark_pages(z));
                        nr_pages += zone_page_state(z, NR_ZONE_INACTIVE_FILE);
                        nr_pages += zone_page_state(z, NR_ZONE_ACTIVE_FILE);
                        x += nr_pages;
                }
        }

        /*
         * Make sure that the number of highmem pages is never larger
         * than the number of the total dirtyable memory. This can only
         * occur in very strange VM situations but we want to make sure
         * that this does not occur.
         */
        return min(x, total);
#else
        return 0;
#endif
}

/**
 * global_dirtyable_memory - number of globally dirtyable pages
 *
 * Return: the global number of pages potentially available for dirty
 * page cache.  This is the base value for the global dirty limits.
 */
static unsigned long global_dirtyable_memory(void)
{
        unsigned long x;

        x = global_zone_page_state(NR_FREE_PAGES);
        /*
         * Pages reserved for the kernel should not be considered
         * dirtyable, to prevent a situation where reclaim has to
         * clean pages in order to balance the zones.
         */
        x -= min(x, totalreserve_pages);

        x += global_node_page_state(NR_INACTIVE_FILE);
        x += global_node_page_state(NR_ACTIVE_FILE);

        if (!vm_highmem_is_dirtyable)
                x -= highmem_dirtyable_memory(x);

        return x + 1;        /* Ensure that we never return 0 */
}

/**
 * domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain
 * @dtc: dirty_throttle_control of interest
 *
 * Calculate @dtc->thresh and ->bg_thresh considering
 * vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}.  The caller
 * must ensure that @dtc->avail is set before calling this function.  The
 * dirty limits will be lifted by 1/4 for real-time tasks.
 */
static void domain_dirty_limits(struct dirty_throttle_control *dtc)
{
        const unsigned long available_memory = dtc->avail;
        struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc);
        unsigned long bytes = vm_dirty_bytes;
        unsigned long bg_bytes = dirty_background_bytes;
        /* convert ratios to per-PAGE_SIZE for higher precision */
        unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100;
        unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100;
        unsigned long thresh;
        unsigned long bg_thresh;
        struct task_struct *tsk;

        /* gdtc is !NULL iff @dtc is for memcg domain */
        if (gdtc) {
                unsigned long global_avail = gdtc->avail;

                /*
                 * The byte settings can't be applied directly to memcg
                 * domains.  Convert them to ratios by scaling against
                 * globally available memory.  As the ratios are in
                 * per-PAGE_SIZE, they can be obtained by dividing bytes by
                 * number of pages.
                 */
                if (bytes)
                        ratio = min(DIV_ROUND_UP(bytes, global_avail),
                                    PAGE_SIZE);
                if (bg_bytes)
                        bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail),
                                       PAGE_SIZE);
                bytes = bg_bytes = 0;
        }

        if (bytes)
                thresh = DIV_ROUND_UP(bytes, PAGE_SIZE);
        else
                thresh = (ratio * available_memory) / PAGE_SIZE;

        if (bg_bytes)
                bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE);
        else
                bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;

        tsk = current;
        if (rt_or_dl_task(tsk)) {
                bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;
                thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;
        }
        /*
         * Dirty throttling logic assumes the limits in page units fit into
         * 32-bits. This gives 16TB dirty limits max which is hopefully enough.
         */
        if (thresh > UINT_MAX)
                thresh = UINT_MAX;
        /* This makes sure bg_thresh is within 32-bits as well */
        if (bg_thresh >= thresh)
                bg_thresh = thresh / 2;
        dtc->thresh = thresh;
        dtc->bg_thresh = bg_thresh;

        /* we should eventually report the domain in the TP */
        if (!gdtc)
                trace_global_dirty_state(bg_thresh, thresh);
}

/**
 * global_dirty_limits - background-writeback and dirty-throttling thresholds
 * @pbackground: out parameter for bg_thresh
 * @pdirty: out parameter for thresh
 *
 * Calculate bg_thresh and thresh for global_wb_domain.  See
 * domain_dirty_limits() for details.
 */
void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
{
        struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB };

        gdtc.avail = global_dirtyable_memory();
        domain_dirty_limits(&gdtc);

        *pbackground = gdtc.bg_thresh;
        *pdirty = gdtc.thresh;
}

/**
 * node_dirty_limit - maximum number of dirty pages allowed in a node
 * @pgdat: the node
 *
 * Return: the maximum number of dirty pages allowed in a node, based
 * on the node's dirtyable memory.
 */
static unsigned long node_dirty_limit(struct pglist_data *pgdat)
{
        unsigned long node_memory = node_dirtyable_memory(pgdat);
        struct task_struct *tsk = current;
        unsigned long dirty;

        if (vm_dirty_bytes)
                dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *
                        node_memory / global_dirtyable_memory();
        else
                dirty = vm_dirty_ratio * node_memory / 100;

        if (rt_or_dl_task(tsk))
                dirty += dirty / 4;

        /*
         * Dirty throttling logic assumes the limits in page units fit into
         * 32-bits. This gives 16TB dirty limits max which is hopefully enough.
         */
        return min_t(unsigned long, dirty, UINT_MAX);
}

/**
 * node_dirty_ok - tells whether a node is within its dirty limits
 * @pgdat: the node to check
 *
 * Return: %true when the dirty pages in @pgdat are within the node's
 * dirty limit, %false if the limit is exceeded.
 */
bool node_dirty_ok(struct pglist_data *pgdat)
{
        unsigned long limit = node_dirty_limit(pgdat);
        unsigned long nr_pages = 0;

        nr_pages += node_page_state(pgdat, NR_FILE_DIRTY);
        nr_pages += node_page_state(pgdat, NR_WRITEBACK);

        return nr_pages <= limit;
}

#ifdef CONFIG_SYSCTL
static int dirty_background_ratio_handler(const struct ctl_table *table, int write,
                void *buffer, size_t *lenp, loff_t *ppos)
{
        int ret;

        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
        if (ret == 0 && write)
                dirty_background_bytes = 0;
        return ret;
}

static int dirty_background_bytes_handler(const struct ctl_table *table, int write,
                void *buffer, size_t *lenp, loff_t *ppos)
{
        int ret;
        unsigned long old_bytes = dirty_background_bytes;

        ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
        if (ret == 0 && write) {
                if (DIV_ROUND_UP(dirty_background_bytes, PAGE_SIZE) >
                                                                UINT_MAX) {
                        dirty_background_bytes = old_bytes;
                        return -ERANGE;
                }
                dirty_background_ratio = 0;
        }
        return ret;
}

static int dirty_ratio_handler(const struct ctl_table *table, int write, void *buffer,
                size_t *lenp, loff_t *ppos)
{
        int old_ratio = vm_dirty_ratio;
        int ret;

        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
        if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
                vm_dirty_bytes = 0;
                writeback_set_ratelimit();
        }
        return ret;
}

static int dirty_bytes_handler(const struct ctl_table *table, int write,
                void *buffer, size_t *lenp, loff_t *ppos)
{
        unsigned long old_bytes = vm_dirty_bytes;
        int ret;

        ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
        if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
                if (DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) > UINT_MAX) {
                        vm_dirty_bytes = old_bytes;
                        return -ERANGE;
                }
                writeback_set_ratelimit();
                vm_dirty_ratio = 0;
        }
        return ret;
}
#endif

static unsigned long wp_next_time(unsigned long cur_time)
{
        cur_time += VM_COMPLETIONS_PERIOD_LEN;
        /* 0 has a special meaning... */
        if (!cur_time)
                return 1;
        return cur_time;
}

static void wb_domain_writeout_add(struct wb_domain *dom,
                                   struct fprop_local_percpu *completions,
                                   unsigned int max_prop_frac, long nr)
{
        __fprop_add_percpu_max(&dom->completions, completions,
                               max_prop_frac, nr);
        /* First event after period switching was turned off? */
        if (unlikely(!dom->period_time)) {
                /*
                 * We can race with other wb_domain_writeout_add calls here but
                 * it does not cause any harm since the resulting time when
                 * timer will fire and what is in writeout_period_time will be
                 * roughly the same.
                 */
                dom->period_time = wp_next_time(jiffies);
                mod_timer(&dom->period_timer, dom->period_time);
        }
}

/*
 * Increment @wb's writeout completion count and the global writeout
 * completion count. Called from __folio_end_writeback().
 */
static inline void __wb_writeout_add(struct bdi_writeback *wb, long nr)
{
        struct wb_domain *cgdom;

        wb_stat_mod(wb, WB_WRITTEN, nr);
        wb_domain_writeout_add(&global_wb_domain, &wb->completions,
                               wb->bdi->max_prop_frac, nr);

        cgdom = mem_cgroup_wb_domain(wb);
        if (cgdom)
                wb_domain_writeout_add(cgdom, wb_memcg_completions(wb),
                                       wb->bdi->max_prop_frac, nr);
}

void wb_writeout_inc(struct bdi_writeback *wb)
{
        unsigned long flags;

        local_irq_save(flags);
        __wb_writeout_add(wb, 1);
        local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(wb_writeout_inc);

/*
 * On idle system, we can be called long after we scheduled because we use
 * deferred timers so count with missed periods.
 */
static void writeout_period(struct timer_list *t)
{
        struct wb_domain *dom = timer_container_of(dom, t, period_timer);
        int miss_periods = (jiffies - dom->period_time) /
                                                 VM_COMPLETIONS_PERIOD_LEN;

        if (fprop_new_period(&dom->completions, miss_periods + 1)) {
                dom->period_time = wp_next_time(dom->period_time +
                                miss_periods * VM_COMPLETIONS_PERIOD_LEN);
                mod_timer(&dom->period_timer, dom->period_time);
        } else {
                /*
                 * Aging has zeroed all fractions. Stop wasting CPU on period
                 * updates.
                 */
                dom->period_time = 0;
        }
}

int wb_domain_init(struct wb_domain *dom, gfp_t gfp)
{
        memset(dom, 0, sizeof(*dom));

        spin_lock_init(&dom->lock);

        timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE);

        dom->dirty_limit_tstamp = jiffies;

        return fprop_global_init(&dom->completions, gfp);
}

#ifdef CONFIG_CGROUP_WRITEBACK
void wb_domain_exit(struct wb_domain *dom)
{
        timer_delete_sync(&dom->period_timer);
        fprop_global_destroy(&dom->completions);
}
#endif

/*
 * bdi_min_ratio keeps the sum of the minimum dirty shares of all
 * registered backing devices, which, for obvious reasons, can not
 * exceed 100%.
 */
static unsigned int bdi_min_ratio;

static int bdi_check_pages_limit(unsigned long pages)
{
        unsigned long max_dirty_pages = global_dirtyable_memory();

        if (pages > max_dirty_pages)
                return -EINVAL;

        return 0;
}

static unsigned long bdi_ratio_from_pages(unsigned long pages)
{
        unsigned long background_thresh;
        unsigned long dirty_thresh;
        unsigned long ratio;

        global_dirty_limits(&background_thresh, &dirty_thresh);
        if (!dirty_thresh)
                return -EINVAL;
        ratio = div64_u64(pages * 100ULL * BDI_RATIO_SCALE, dirty_thresh);

        return ratio;
}

static u64 bdi_get_bytes(unsigned int ratio)
{
        unsigned long background_thresh;
        unsigned long dirty_thresh;
        u64 bytes;

        global_dirty_limits(&background_thresh, &dirty_thresh);
        bytes = (dirty_thresh * PAGE_SIZE * ratio) / BDI_RATIO_SCALE / 100;

        return bytes;
}

static int __bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
{
        unsigned int delta;
        int ret = 0;

        if (min_ratio > 100 * BDI_RATIO_SCALE)
                return -EINVAL;

        spin_lock_bh(&bdi_lock);
        if (min_ratio > bdi->max_ratio) {
                ret = -EINVAL;
        } else {
                if (min_ratio < bdi->min_ratio) {
                        delta = bdi->min_ratio - min_ratio;
                        bdi_min_ratio -= delta;
                        bdi->min_ratio = min_ratio;
                } else {
                        delta = min_ratio - bdi->min_ratio;
                        if (bdi_min_ratio + delta < 100 * BDI_RATIO_SCALE) {
                                bdi_min_ratio += delta;
                                bdi->min_ratio = min_ratio;
                        } else {
                                ret = -EINVAL;
                        }
                }
        }
        spin_unlock_bh(&bdi_lock);

        return ret;
}

static int __bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio)
{
        int ret = 0;

        if (max_ratio > 100 * BDI_RATIO_SCALE)
                return -EINVAL;

        spin_lock_bh(&bdi_lock);
        if (bdi->min_ratio > max_ratio) {
                ret = -EINVAL;
        } else {
                bdi->max_ratio = max_ratio;
                bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) /
                                                (100 * BDI_RATIO_SCALE);
        }
        spin_unlock_bh(&bdi_lock);

        return ret;
}

int bdi_set_min_ratio_no_scale(struct backing_dev_info *bdi, unsigned int min_ratio)
{
        return __bdi_set_min_ratio(bdi, min_ratio);
}

int bdi_set_max_ratio_no_scale(struct backing_dev_info *bdi, unsigned int max_ratio)
{
        return __bdi_set_max_ratio(bdi, max_ratio);
}

int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
{
        return __bdi_set_min_ratio(bdi, min_ratio * BDI_RATIO_SCALE);
}

int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio)
{
        return __bdi_set_max_ratio(bdi, max_ratio * BDI_RATIO_SCALE);
}
EXPORT_SYMBOL(bdi_set_max_ratio);

u64 bdi_get_min_bytes(struct backing_dev_info *bdi)
{
        return bdi_get_bytes(bdi->min_ratio);
}

int bdi_set_min_bytes(struct backing_dev_info *bdi, u64 min_bytes)
{
        int ret;
        unsigned long pages = min_bytes >> PAGE_SHIFT;
        long min_ratio;

        ret = bdi_check_pages_limit(pages);
        if (ret)
                return ret;

        min_ratio = bdi_ratio_from_pages(pages);
        if (min_ratio < 0)
                return min_ratio;
        return __bdi_set_min_ratio(bdi, min_ratio);
}

u64 bdi_get_max_bytes(struct backing_dev_info *bdi)
{
        return bdi_get_bytes(bdi->max_ratio);
}

int bdi_set_max_bytes(struct backing_dev_info *bdi, u64 max_bytes)
{
        int ret;
        unsigned long pages = max_bytes >> PAGE_SHIFT;
        long max_ratio;

        ret = bdi_check_pages_limit(pages);
        if (ret)
                return ret;

        max_ratio = bdi_ratio_from_pages(pages);
        if (max_ratio < 0)
                return max_ratio;
        return __bdi_set_max_ratio(bdi, max_ratio);
}

int bdi_set_strict_limit(struct backing_dev_info *bdi, unsigned int strict_limit)
{
        if (strict_limit > 1)
                return -EINVAL;

        spin_lock_bh(&bdi_lock);
        if (strict_limit)
                bdi->capabilities |= BDI_CAP_STRICTLIMIT;
        else
                bdi->capabilities &= ~BDI_CAP_STRICTLIMIT;
        spin_unlock_bh(&bdi_lock);

        return 0;
}

static unsigned long dirty_freerun_ceiling(unsigned long thresh,
                                           unsigned long bg_thresh)
{
        return (thresh + bg_thresh) / 2;
}

static unsigned long hard_dirty_limit(struct wb_domain *dom,
                                      unsigned long thresh)
{
        return max(thresh, dom->dirty_limit);
}

/*
 * Memory which can be further allocated to a memcg domain is capped by
 * system-wide clean memory excluding the amount being used in the domain.
 */
static void mdtc_calc_avail(struct dirty_throttle_control *mdtc,
                            unsigned long filepages, unsigned long headroom)
{
        struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc);
        unsigned long clean = filepages - min(filepages, mdtc->dirty);
        unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty);
        unsigned long other_clean = global_clean - min(global_clean, clean);

        mdtc->avail = filepages + min(headroom, other_clean);
}

static inline bool dtc_is_global(struct dirty_throttle_control *dtc)
{
        return mdtc_gdtc(dtc) == NULL;
}

/*
 * Dirty background will ignore pages being written as we're trying to
 * decide whether to put more under writeback.
 */
static void domain_dirty_avail(struct dirty_throttle_control *dtc,
                               bool include_writeback)
{
        if (dtc_is_global(dtc)) {
                dtc->avail = global_dirtyable_memory();
                dtc->dirty = global_node_page_state(NR_FILE_DIRTY);
                if (include_writeback)
                        dtc->dirty += global_node_page_state(NR_WRITEBACK);
        } else {
                unsigned long filepages = 0, headroom = 0, writeback = 0;

                mem_cgroup_wb_stats(dtc->wb, &filepages, &headroom, &dtc->dirty,
                                    &writeback);
                if (include_writeback)
                        dtc->dirty += writeback;
                mdtc_calc_avail(dtc, filepages, headroom);
        }
}

/**
 * __wb_calc_thresh - @wb's share of dirty threshold
 * @dtc: dirty_throttle_context of interest
 * @thresh: dirty throttling or dirty background threshold of wb_domain in @dtc
 *
 * Note that balance_dirty_pages() will only seriously take dirty throttling
 * threshold as a hard limit when sleeping max_pause per page is not enough
 * to keep the dirty pages under control. For example, when the device is
 * completely stalled due to some error conditions, or when there are 1000
 * dd tasks writing to a slow 10MB/s USB key.
 * In the other normal situations, it acts more gently by throttling the tasks
 * more (rather than completely block them) when the wb dirty pages go high.
 *
 * It allocates high/low dirty limits to fast/slow devices, in order to prevent
 * - starving fast devices
 * - piling up dirty pages (that will take long time to sync) on slow devices
 *
 * The wb's share of dirty limit will be adapting to its throughput and
 * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
 *
 * Return: @wb's dirty limit in pages. For dirty throttling limit, the term
 * "dirty" in the context of dirty balancing includes all PG_dirty and
 * PG_writeback pages.
 */
static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc,
                                      unsigned long thresh)
{
        struct wb_domain *dom = dtc_dom(dtc);
        struct bdi_writeback *wb = dtc->wb;
        u64 wb_thresh;
        u64 wb_max_thresh;
        unsigned long numerator, denominator;
        unsigned long wb_min_ratio, wb_max_ratio;

        /*
         * Calculate this wb's share of the thresh ratio.
         */
        fprop_fraction_percpu(&dom->completions, dtc->wb_completions,
                              &numerator, &denominator);

        wb_thresh = (thresh * (100 * BDI_RATIO_SCALE - bdi_min_ratio)) / (100 * BDI_RATIO_SCALE);
        wb_thresh *= numerator;
        wb_thresh = div64_ul(wb_thresh, denominator);

        wb_min_max_ratio(wb, &wb_min_ratio, &wb_max_ratio);

        wb_thresh += (thresh * wb_min_ratio) / (100 * BDI_RATIO_SCALE);

        /*
         * It's very possible that wb_thresh is close to 0 not because the
         * device is slow, but that it has remained inactive for long time.
         * Honour such devices a reasonable good (hopefully IO efficient)
         * threshold, so that the occasional writes won't be blocked and active
         * writes can rampup the threshold quickly.
         */
        if (thresh > dtc->dirty) {
                if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT))
                        wb_thresh = max(wb_thresh, (thresh - dtc->dirty) / 100);
                else
                        wb_thresh = max(wb_thresh, (thresh - dtc->dirty) / 8);
        }

        wb_max_thresh = thresh * wb_max_ratio / (100 * BDI_RATIO_SCALE);
        if (wb_thresh > wb_max_thresh)
                wb_thresh = wb_max_thresh;

        return wb_thresh;
}

unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh)
{
        struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };

        domain_dirty_avail(&gdtc, true);
        return __wb_calc_thresh(&gdtc, thresh);
}

unsigned long cgwb_calc_thresh(struct bdi_writeback *wb)
{
        struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB };
        struct dirty_throttle_control mdtc = { MDTC_INIT(wb, &gdtc) };

        domain_dirty_avail(&gdtc, true);
        domain_dirty_avail(&mdtc, true);
        domain_dirty_limits(&mdtc);

        return __wb_calc_thresh(&mdtc, mdtc.thresh);
}

/*
 *                           setpoint - dirty 3
 *        f(dirty) := 1.0 + (----------------)
 *                           limit - setpoint
 *
 * it's a 3rd order polynomial that subjects to
 *
 * (1) f(freerun)  = 2.0 => rampup dirty_ratelimit reasonably fast
 * (2) f(setpoint) = 1.0 => the balance point
 * (3) f(limit)    = 0   => the hard limit
 * (4) df/dx      <= 0         => negative feedback control
 * (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
 *     => fast response on large errors; small oscillation near setpoint
 */
static long long pos_ratio_polynom(unsigned long setpoint,
                                          unsigned long dirty,
                                          unsigned long limit)
{
        long long pos_ratio;
        long x;

        x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT,
                      (limit - setpoint) | 1);
        pos_ratio = x;
        pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
        pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
        pos_ratio += 1 << RATELIMIT_CALC_SHIFT;

        return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT);
}

/*
 * Dirty position control.
 *
 * (o) global/bdi setpoints
 *
 * We want the dirty pages be balanced around the global/wb setpoints.
 * When the number of dirty pages is higher/lower than the setpoint, the
 * dirty position control ratio (and hence task dirty ratelimit) will be
 * decreased/increased to bring the dirty pages back to the setpoint.
 *
 *     pos_ratio = 1 << RATELIMIT_CALC_SHIFT
 *
 *     if (dirty < setpoint) scale up   pos_ratio
 *     if (dirty > setpoint) scale down pos_ratio
 *
 *     if (wb_dirty < wb_setpoint) scale up   pos_ratio
 *     if (wb_dirty > wb_setpoint) scale down pos_ratio
 *
 *     task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
 *
 * (o) global control line
 *
 *     ^ pos_ratio
 *     |
 *     |            |<===== global dirty control scope ======>|
 * 2.0  * * * * * * *
 *     |            .*
 *     |            . *
 *     |            .   *
 *     |            .     *
 *     |            .        *
 *     |            .            *
 * 1.0 ................................*
 *     |            .                  .     *
 *     |            .                  .          *
 *     |            .                  .              *
 *     |            .                  .                 *
 *     |            .                  .                    *
 *   0 +------------.------------------.----------------------*------------->
 *           freerun^          setpoint^                 limit^   dirty pages
 *
 * (o) wb control line
 *
 *     ^ pos_ratio
 *     |
 *     |            *
 *     |              *
 *     |                *
 *     |                  *
 *     |                    * |<=========== span ============>|
 * 1.0 .......................*
 *     |                      . *
 *     |                      .   *
 *     |                      .     *
 *     |                      .       *
 *     |                      .         *
 *     |                      .           *
 *     |                      .             *
 *     |                      .               *
 *     |                      .                 *
 *     |                      .                   *
 *     |                      .                     *
 * 1/4 ...............................................* * * * * * * * * * * *
 *     |                      .                         .
 *     |                      .                           .
 *     |                      .                             .
 *   0 +----------------------.-------------------------------.------------->
 *                wb_setpoint^                    x_intercept^
 *
 * The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can
 * be smoothly throttled down to normal if it starts high in situations like
 * - start writing to a slow SD card and a fast disk at the same time. The SD
 *   card's wb_dirty may rush to many times higher than wb_setpoint.
 * - the wb dirty thresh drops quickly due to change of JBOD workload
 */
static void wb_position_ratio(struct dirty_throttle_control *dtc)
{
        struct bdi_writeback *wb = dtc->wb;
        unsigned long write_bw = READ_ONCE(wb->avg_write_bandwidth);
        unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);
        unsigned long limit = dtc->limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);
        unsigned long wb_thresh = dtc->wb_thresh;
        unsigned long x_intercept;
        unsigned long setpoint;                /* dirty pages' target balance point */
        unsigned long wb_setpoint;
        unsigned long span;
        long long pos_ratio;                /* for scaling up/down the rate limit */
        long x;

        dtc->pos_ratio = 0;

        if (unlikely(dtc->dirty >= limit))
                return;

        /*
         * global setpoint
         *
         * See comment for pos_ratio_polynom().
         */
        setpoint = (freerun + limit) / 2;
        pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit);

        /*
         * The strictlimit feature is a tool preventing mistrusted filesystems
         * from growing a large number of dirty pages before throttling. For
         * such filesystems balance_dirty_pages always checks wb counters
         * against wb limits. Even if global "nr_dirty" is under "freerun".
         * This is especially important for fuse which sets bdi->max_ratio to
         * 1% by default.
         *
         * Here, in wb_position_ratio(), we calculate pos_ratio based on
         * two values: wb_dirty and wb_thresh. Let's consider an example:
         * total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global
         * limits are set by default to 10% and 20% (background and throttle).
         * Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages.
         * wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is
         * about ~6K pages (as the average of background and throttle wb
         * limits). The 3rd order polynomial will provide positive feedback if
         * wb_dirty is under wb_setpoint and vice versa.
         *
         * Note, that we cannot use global counters in these calculations
         * because we want to throttle process writing to a strictlimit wb
         * much earlier than global "freerun" is reached (~23MB vs. ~2.3GB
         * in the example above).
         */
        if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
                long long wb_pos_ratio;

                if (dtc->wb_dirty >= wb_thresh)
                        return;

                wb_setpoint = dirty_freerun_ceiling(wb_thresh,
                                                    dtc->wb_bg_thresh);

                if (wb_setpoint == 0 || wb_setpoint == wb_thresh)
                        return;

                wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty,
                                                 wb_thresh);

                /*
                 * Typically, for strictlimit case, wb_setpoint << setpoint
                 * and pos_ratio >> wb_pos_ratio. In the other words global
                 * state ("dirty") is not limiting factor and we have to
                 * make decision based on wb counters. But there is an
                 * important case when global pos_ratio should get precedence:
                 * global limits are exceeded (e.g. due to activities on other
                 * wb's) while given strictlimit wb is below limit.
                 *
                 * "pos_ratio * wb_pos_ratio" would work for the case above,
                 * but it would look too non-natural for the case of all
                 * activity in the system coming from a single strictlimit wb
                 * with bdi->max_ratio == 100%.
                 *
                 * Note that min() below somewhat changes the dynamics of the
                 * control system. Normally, pos_ratio value can be well over 3
                 * (when globally we are at freerun and wb is well below wb
                 * setpoint). Now the maximum pos_ratio in the same situation
                 * is 2. We might want to tweak this if we observe the control
                 * system is too slow to adapt.
                 */
                dtc->pos_ratio = min(pos_ratio, wb_pos_ratio);
                return;
        }

        /*
         * We have computed basic pos_ratio above based on global situation. If
         * the wb is over/under its share of dirty pages, we want to scale
         * pos_ratio further down/up. That is done by the following mechanism.
         */

        /*
         * wb setpoint
         *
         *        f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint)
         *
         *                        x_intercept - wb_dirty
         *                     := --------------------------
         *                        x_intercept - wb_setpoint
         *
         * The main wb control line is a linear function that subjects to
         *
         * (1) f(wb_setpoint) = 1.0
         * (2) k = - 1 / (8 * write_bw)  (in single wb case)
         *     or equally: x_intercept = wb_setpoint + 8 * write_bw
         *
         * For single wb case, the dirty pages are observed to fluctuate
         * regularly within range
         *        [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2]
         * for various filesystems, where (2) can yield in a reasonable 12.5%
         * fluctuation range for pos_ratio.
         *
         * For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its
         * own size, so move the slope over accordingly and choose a slope that
         * yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh.
         */
        if (unlikely(wb_thresh > dtc->thresh))
                wb_thresh = dtc->thresh;
        /*
         * scale global setpoint to wb's:
         *        wb_setpoint = setpoint * wb_thresh / thresh
         */
        x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1);
        wb_setpoint = setpoint * (u64)x >> 16;
        /*
         * Use span=(8*write_bw) in single wb case as indicated by
         * (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case.
         *
         *        wb_thresh                    thresh - wb_thresh
         * span = --------- * (8 * write_bw) + ------------------ * wb_thresh
         *         thresh                           thresh
         */
        span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16;
        x_intercept = wb_setpoint + span;

        if (dtc->wb_dirty < x_intercept - span / 4) {
                pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty),
                                      (x_intercept - wb_setpoint) | 1);
        } else
                pos_ratio /= 4;

        /*
         * wb reserve area, safeguard against dirty pool underrun and disk idle
         * It may push the desired control point of global dirty pages higher
         * than setpoint.
         */
        x_intercept = wb_thresh / 2;
        if (dtc->wb_dirty < x_intercept) {
                if (dtc->wb_dirty > x_intercept / 8)
                        pos_ratio = div_u64(pos_ratio * x_intercept,
                                            dtc->wb_dirty);
                else
                        pos_ratio *= 8;
        }

        dtc->pos_ratio = pos_ratio;
}

static void wb_update_write_bandwidth(struct bdi_writeback *wb,
                                      unsigned long elapsed,
                                      unsigned long written)
{
        const unsigned long period = roundup_pow_of_two(3 * HZ);
        unsigned long avg = wb->avg_write_bandwidth;
        unsigned long old = wb->write_bandwidth;
        u64 bw;

        /*
         * bw = written * HZ / elapsed
         *
         *                   bw * elapsed + write_bandwidth * (period - elapsed)
         * write_bandwidth = ---------------------------------------------------
         *                                          period
         *
         * @written may have decreased due to folio_redirty_for_writepage().
         * Avoid underflowing @bw calculation.
         */
        bw = written - min(written, wb->written_stamp);
        bw *= HZ;
        if (unlikely(elapsed > period)) {
                bw = div64_ul(bw, elapsed);
                avg = bw;
                goto out;
        }
        bw += (u64)wb->write_bandwidth * (period - elapsed);
        bw >>= ilog2(period);

        /*
         * one more level of smoothing, for filtering out sudden spikes
         */
        if (avg > old && old >= (unsigned long)bw)
                avg -= (avg - old) >> 3;

        if (avg < old && old <= (unsigned long)bw)
                avg += (old - avg) >> 3;

out:
        /* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */
        avg = max(avg, 1LU);
        if (wb_has_dirty_io(wb)) {
                long delta = avg - wb->avg_write_bandwidth;
                WARN_ON_ONCE(atomic_long_add_return(delta,
                                        &wb->bdi->tot_write_bandwidth) <= 0);
        }
        wb->write_bandwidth = bw;
        WRITE_ONCE(wb->avg_write_bandwidth, avg);
}

static void update_dirty_limit(struct dirty_throttle_control *dtc)
{
        struct wb_domain *dom = dtc_dom(dtc);
        unsigned long thresh = dtc->thresh;
        unsigned long limit = dom->dirty_limit;

        /*
         * Follow up in one step.
         */
        if (limit < thresh) {
                limit = thresh;
                goto update;
        }

        /*
         * Follow down slowly. Use the higher one as the target, because thresh
         * may drop below dirty. This is exactly the reason to introduce
         * dom->dirty_limit which is guaranteed to lie above the dirty pages.
         */
        thresh = max(thresh, dtc->dirty);
        if (limit > thresh) {
                limit -= (limit - thresh) >> 5;
                goto update;
        }
        return;
update:
        dom->dirty_limit = limit;
}

static void domain_update_dirty_limit(struct dirty_throttle_control *dtc,
                                      unsigned long now)
{
        struct wb_domain *dom = dtc_dom(dtc);

        /*
         * check locklessly first to optimize away locking for the most time
         */
        if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL))
                return;

        spin_lock(&dom->lock);
        if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) {
                update_dirty_limit(dtc);
                dom->dirty_limit_tstamp = now;
        }
        spin_unlock(&dom->lock);
}

/*
 * Maintain wb->dirty_ratelimit, the base dirty throttle rate.
 *
 * Normal wb tasks will be curbed at or below it in long term.
 * Obviously it should be around (write_bw / N) when there are N dd tasks.
 */
static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc,
                                      unsigned long dirtied,
                                      unsigned long elapsed)
{
        struct bdi_writeback *wb = dtc->wb;
        unsigned long dirty = dtc->dirty;
        unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);
        unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);
        unsigned long setpoint = (freerun + limit) / 2;
        unsigned long write_bw = wb->avg_write_bandwidth;
        unsigned long dirty_ratelimit = wb->dirty_ratelimit;
        unsigned long dirty_rate;
        unsigned long task_ratelimit;
        unsigned long balanced_dirty_ratelimit;
        unsigned long step;
        unsigned long x;
        unsigned long shift;

        /*
         * The dirty rate will match the writeout rate in long term, except
         * when dirty pages are truncated by userspace or re-dirtied by FS.
         */
        dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed;

        /*
         * task_ratelimit reflects each dd's dirty rate for the past 200ms.
         */
        task_ratelimit = (u64)dirty_ratelimit *
                                        dtc->pos_ratio >> RATELIMIT_CALC_SHIFT;
        task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */

        /*
         * A linear estimation of the "balanced" throttle rate. The theory is,
         * if there are N dd tasks, each throttled at task_ratelimit, the wb's
         * dirty_rate will be measured to be (N * task_ratelimit). So the below
         * formula will yield the balanced rate limit (write_bw / N).
         *
         * Note that the expanded form is not a pure rate feedback:
         *        rate_(i+1) = rate_(i) * (write_bw / dirty_rate)                     (1)
         * but also takes pos_ratio into account:
         *        rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio  (2)
         *
         * (1) is not realistic because pos_ratio also takes part in balancing
         * the dirty rate.  Consider the state
         *        pos_ratio = 0.5                                                     (3)
         *        rate = 2 * (write_bw / N)                                     (4)
         * If (1) is used, it will stuck in that state! Because each dd will
         * be throttled at
         *        task_ratelimit = pos_ratio * rate = (write_bw / N)             (5)
         * yielding
         *        dirty_rate = N * task_ratelimit = write_bw                     (6)
         * put (6) into (1) we get
         *        rate_(i+1) = rate_(i)                                             (7)
         *
         * So we end up using (2) to always keep
         *        rate_(i+1) ~= (write_bw / N)                                     (8)
         * regardless of the value of pos_ratio. As long as (8) is satisfied,
         * pos_ratio is able to drive itself to 1.0, which is not only where
         * the dirty count meet the setpoint, but also where the slope of
         * pos_ratio is most flat and hence task_ratelimit is least fluctuated.
         */
        balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw,
                                           dirty_rate | 1);
        /*
         * balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw
         */
        if (unlikely(balanced_dirty_ratelimit > write_bw))
                balanced_dirty_ratelimit = write_bw;

        /*
         * We could safely do this and return immediately:
         *
         *        wb->dirty_ratelimit = balanced_dirty_ratelimit;
         *
         * However to get a more stable dirty_ratelimit, the below elaborated
         * code makes use of task_ratelimit to filter out singular points and
         * limit the step size.
         *
         * The below code essentially only uses the relative value of
         *
         *        task_ratelimit - dirty_ratelimit
         *        = (pos_ratio - 1) * dirty_ratelimit
         *
         * which reflects the direction and size of dirty position error.
         */

        /*
         * dirty_ratelimit will follow balanced_dirty_ratelimit iff
         * task_ratelimit is on the same side of dirty_ratelimit, too.
         * For example, when
         * - dirty_ratelimit > balanced_dirty_ratelimit
         * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint)
         * lowering dirty_ratelimit will help meet both the position and rate
         * control targets. Otherwise, don't update dirty_ratelimit if it will
         * only help meet the rate target. After all, what the users ultimately
         * feel and care are stable dirty rate and small position error.
         *
         * |task_ratelimit - dirty_ratelimit| is used to limit the step size
         * and filter out the singular points of balanced_dirty_ratelimit. Which
         * keeps jumping around randomly and can even leap far away at times
         * due to the small 200ms estimation period of dirty_rate (we want to
         * keep that period small to reduce time lags).
         */
        step = 0;

        /*
         * For strictlimit case, calculations above were based on wb counters
         * and limits (starting from pos_ratio = wb_position_ratio() and up to
         * balanced_dirty_ratelimit = task_ratelimit * write_bw / dirty_rate).
         * Hence, to calculate "step" properly, we have to use wb_dirty as
         * "dirty" and wb_setpoint as "setpoint".
         */
        if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
                dirty = dtc->wb_dirty;
                setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2;
        }

        if (dirty < setpoint) {
                x = min3(wb->balanced_dirty_ratelimit,
                         balanced_dirty_ratelimit, task_ratelimit);
                if (dirty_ratelimit < x)
                        step = x - dirty_ratelimit;
        } else {
                x = max3(wb->balanced_dirty_ratelimit,
                         balanced_dirty_ratelimit, task_ratelimit);
                if (dirty_ratelimit > x)
                        step = dirty_ratelimit - x;
        }

        /*
         * Don't pursue 100% rate matching. It's impossible since the balanced
         * rate itself is constantly fluctuating. So decrease the track speed
         * when it gets close to the target. Helps eliminate pointless tremors.
         */
        shift = dirty_ratelimit / (2 * step + 1);
        if (shift < BITS_PER_LONG)
                step = DIV_ROUND_UP(step >> shift, 8);
        else
                step = 0;

        if (dirty_ratelimit < balanced_dirty_ratelimit)
                dirty_ratelimit += step;
        else
                dirty_ratelimit -= step;

        WRITE_ONCE(wb->dirty_ratelimit, max(dirty_ratelimit, 1UL));
        wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit;

        trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit);
}

static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc,
                                  struct dirty_throttle_control *mdtc,
                                  bool update_ratelimit)
{
        struct bdi_writeback *wb = gdtc->wb;
        unsigned long now = jiffies;
        unsigned long elapsed;
        unsigned long dirtied;
        unsigned long written;

        spin_lock(&wb->list_lock);

        /*
         * Lockless checks for elapsed time are racy and delayed update after
         * IO completion doesn't do it at all (to make sure written pages are
         * accounted reasonably quickly). Make sure elapsed >= 1 to avoid
         * division errors.
         */
        elapsed = max(now - wb->bw_time_stamp, 1UL);
        dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]);
        written = percpu_counter_read(&wb->stat[WB_WRITTEN]);

        if (update_ratelimit) {
                domain_update_dirty_limit(gdtc, now);
                wb_update_dirty_ratelimit(gdtc, dirtied, elapsed);

                /*
                 * @mdtc is always NULL if !CGROUP_WRITEBACK but the
                 * compiler has no way to figure that out.  Help it.
                 */
                if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) {
                        domain_update_dirty_limit(mdtc, now);
                        wb_update_dirty_ratelimit(mdtc, dirtied, elapsed);
                }
        }
        wb_update_write_bandwidth(wb, elapsed, written);

        wb->dirtied_stamp = dirtied;
        wb->written_stamp = written;
        WRITE_ONCE(wb->bw_time_stamp, now);
        spin_unlock(&wb->list_lock);
}

void wb_update_bandwidth(struct bdi_writeback *wb)
{
        struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };

        __wb_update_bandwidth(&gdtc, NULL, false);
}

/* Interval after which we consider wb idle and don't estimate bandwidth */
#define WB_BANDWIDTH_IDLE_JIF (HZ)

static void wb_bandwidth_estimate_start(struct bdi_writeback *wb)
{
        unsigned long now = jiffies;
        unsigned long elapsed = now - READ_ONCE(wb->bw_time_stamp);

        if (elapsed > WB_BANDWIDTH_IDLE_JIF &&
            !atomic_read(&wb->writeback_inodes)) {
                spin_lock(&wb->list_lock);
                wb->dirtied_stamp = wb_stat(wb, WB_DIRTIED);
                wb->written_stamp = wb_stat(wb, WB_WRITTEN);
                WRITE_ONCE(wb->bw_time_stamp, now);
                spin_unlock(&wb->list_lock);
        }
}

/*
 * After a task dirtied this many pages, balance_dirty_pages_ratelimited()
 * will look to see if it needs to start dirty throttling.
 *
 * If dirty_poll_interval is too low, big NUMA machines will call the expensive
 * global_zone_page_state() too often. So scale it near-sqrt to the safety margin
 * (the number of pages we may dirty without exceeding the dirty limits).
 */
static unsigned long dirty_poll_interval(unsigned long dirty,
                                         unsigned long thresh)
{
        if (thresh > dirty)
                return 1UL << (ilog2(thresh - dirty) >> 1);

        return 1;
}

static unsigned long wb_max_pause(struct bdi_writeback *wb,
                                  unsigned long wb_dirty)
{
        unsigned long bw = READ_ONCE(wb->avg_write_bandwidth);
        unsigned long t;

        /*
         * Limit pause time for small memory systems. If sleeping for too long
         * time, a small pool of dirty/writeback pages may go empty and disk go
         * idle.
         *
         * 8 serves as the safety ratio.
         */
        t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8));
        t++;

        return min_t(unsigned long, t, MAX_PAUSE);
}

static long wb_min_pause(struct bdi_writeback *wb,
                         long max_pause,
                         unsigned long task_ratelimit,
                         unsigned long dirty_ratelimit,
                         int *nr_dirtied_pause)
{
        long hi = ilog2(READ_ONCE(wb->avg_write_bandwidth));
        long lo = ilog2(READ_ONCE(wb->dirty_ratelimit));
        long t;                /* target pause */
        long pause;        /* estimated next pause */
        int pages;        /* target nr_dirtied_pause */

        /* target for 10ms pause on 1-dd case */
        t = max(1, HZ / 100);

        /*
         * Scale up pause time for concurrent dirtiers in order to reduce CPU
         * overheads.
         *
         * (N * 10ms) on 2^N concurrent tasks.
         */
        if (hi > lo)
                t += (hi - lo) * (10 * HZ) / 1024;

        /*
         * This is a bit convoluted. We try to base the next nr_dirtied_pause
         * on the much more stable dirty_ratelimit. However the next pause time
         * will be computed based on task_ratelimit and the two rate limits may
         * depart considerably at some time. Especially if task_ratelimit goes
         * below dirty_ratelimit/2 and the target pause is max_pause, the next
         * pause time will be max_pause*2 _trimmed down_ to max_pause.  As a
         * result task_ratelimit won't be executed faithfully, which could
         * eventually bring down dirty_ratelimit.
         *
         * We apply two rules to fix it up:
         * 1) try to estimate the next pause time and if necessary, use a lower
         *    nr_dirtied_pause so as not to exceed max_pause. When this happens,
         *    nr_dirtied_pause will be "dancing" with task_ratelimit.
         * 2) limit the target pause time to max_pause/2, so that the normal
         *    small fluctuations of task_ratelimit won't trigger rule (1) and
         *    nr_dirtied_pause will remain as stable as dirty_ratelimit.
         */
        t = min(t, 1 + max_pause / 2);
        pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);

        /*
         * Tiny nr_dirtied_pause is found to hurt I/O performance in the test
         * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}.
         * When the 16 consecutive reads are often interrupted by some dirty
         * throttling pause during the async writes, cfq will go into idles
         * (deadline is fine). So push nr_dirtied_pause as high as possible
         * until reaches DIRTY_POLL_THRESH=32 pages.
         */
        if (pages < DIRTY_POLL_THRESH) {
                t = max_pause;
                pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
                if (pages > DIRTY_POLL_THRESH) {
                        pages = DIRTY_POLL_THRESH;
                        t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit;
                }
        }

        pause = HZ * pages / (task_ratelimit + 1);
        if (pause > max_pause) {
                t = max_pause;
                pages = task_ratelimit * t / roundup_pow_of_two(HZ);
        }

        *nr_dirtied_pause = pages;
        /*
         * The minimal pause time will normally be half the target pause time.
         */
        return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t;
}

static inline void wb_dirty_limits(struct dirty_throttle_control *dtc)
{
        struct bdi_writeback *wb = dtc->wb;
        unsigned long wb_reclaimable;

        /*
         * wb_thresh is not treated as some limiting factor as
         * dirty_thresh, due to reasons
         * - in JBOD setup, wb_thresh can fluctuate a lot
         * - in a system with HDD and USB key, the USB key may somehow
         *   go into state (wb_dirty >> wb_thresh) either because
         *   wb_dirty starts high, or because wb_thresh drops low.
         *   In this case we don't want to hard throttle the USB key
         *   dirtiers for 100 seconds until wb_dirty drops under
         *   wb_thresh. Instead the auxiliary wb control line in
         *   wb_position_ratio() will let the dirtier task progress
         *   at some rate <= (write_bw / 2) for bringing down wb_dirty.
         */
        dtc->wb_thresh = __wb_calc_thresh(dtc, dtc->thresh);
        dtc->wb_bg_thresh = dtc->thresh ?
                div_u64((u64)dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0;

        /*
         * In order to avoid the stacked BDI deadlock we need
         * to ensure we accurately count the 'dirty' pages when
         * the threshold is low.
         *
         * Otherwise it would be possible to get thresh+n pages
         * reported dirty, even though there are thresh-m pages
         * actually dirty; with m+n sitting in the percpu
         * deltas.
         */
        if (dtc->wb_thresh < 2 * wb_stat_error()) {
                wb_reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE);
                dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK);
        } else {
                wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE);
                dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK);
        }
}

static unsigned long domain_poll_intv(struct dirty_throttle_control *dtc,
                                      bool strictlimit)
{
        unsigned long dirty, thresh;

        if (strictlimit) {
                dirty = dtc->wb_dirty;
                thresh = dtc->wb_thresh;
        } else {
                dirty = dtc->dirty;
                thresh = dtc->thresh;
        }

        return dirty_poll_interval(dirty, thresh);
}

/*
 * Throttle it only when the background writeback cannot catch-up. This avoids
 * (excessively) small writeouts when the wb limits are ramping up in case of
 * !strictlimit.
 *
 * In strictlimit case make decision based on the wb counters and limits. Small
 * writeouts when the wb limits are ramping up are the price we consciously pay
 * for strictlimit-ing.
 */
static void domain_dirty_freerun(struct dirty_throttle_control *dtc,
                                 bool strictlimit)
{
        unsigned long dirty, thresh, bg_thresh;

        if (unlikely(strictlimit)) {
                wb_dirty_limits(dtc);
                dirty = dtc->wb_dirty;
                thresh = dtc->wb_thresh;
                bg_thresh = dtc->wb_bg_thresh;
        } else {
                dirty = dtc->dirty;
                thresh = dtc->thresh;
                bg_thresh = dtc->bg_thresh;
        }
        dtc->freerun = dirty <= dirty_freerun_ceiling(thresh, bg_thresh);
}

static void balance_domain_limits(struct dirty_throttle_control *dtc,
                                  bool strictlimit)
{
        domain_dirty_avail(dtc, true);
        domain_dirty_limits(dtc);
        domain_dirty_freerun(dtc, strictlimit);
}

static void wb_dirty_freerun(struct dirty_throttle_control *dtc,
                             bool strictlimit)
{
        dtc->freerun = false;

        /* was already handled in domain_dirty_freerun */
        if (strictlimit)
                return;

        wb_dirty_limits(dtc);
        /*
         * LOCAL_THROTTLE tasks must not be throttled when below the per-wb
         * freerun ceiling.
         */
        if (!(current->flags & PF_LOCAL_THROTTLE))
                return;

        dtc->freerun = dtc->wb_dirty <
                       dirty_freerun_ceiling(dtc->wb_thresh, dtc->wb_bg_thresh);
}

static inline void wb_dirty_exceeded(struct dirty_throttle_control *dtc,
                                     bool strictlimit)
{
        dtc->dirty_exceeded = (dtc->wb_dirty > dtc->wb_thresh) &&
                ((dtc->dirty > dtc->thresh) || strictlimit);
}

/*
 * The limits fields dirty_exceeded and pos_ratio won't be updated if wb is
 * in freerun state. Please don't use these invalid fields in freerun case.
 */
static void balance_wb_limits(struct dirty_throttle_control *dtc,
                              bool strictlimit)
{
        wb_dirty_freerun(dtc, strictlimit);
        if (dtc->freerun)
                return;

        wb_dirty_exceeded(dtc, strictlimit);
        wb_position_ratio(dtc);
}

/*
 * balance_dirty_pages() must be called by processes which are generating dirty
 * data.  It looks at the number of dirty pages in the machine and will force
 * the caller to wait once crossing the (background_thresh + dirty_thresh) / 2.
 * If we're over `background_thresh' then the writeback threads are woken to
 * perform some writeout.
 */
static int balance_dirty_pages(struct bdi_writeback *wb,
                               unsigned long pages_dirtied, unsigned int flags)
{
        struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) };
        struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) };
        struct dirty_throttle_control * const gdtc = &gdtc_stor;
        struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ?
                                                     &mdtc_stor : NULL;
        struct dirty_throttle_control *sdtc;
        unsigned long nr_dirty;
        long period;
        long pause;
        long max_pause;
        long min_pause;
        int nr_dirtied_pause;
        unsigned long task_ratelimit;
        unsigned long dirty_ratelimit;
        struct backing_dev_info *bdi = wb->bdi;
        bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT;
        unsigned long start_time = jiffies;
        int ret = 0;

        for (;;) {
                unsigned long now = jiffies;

                nr_dirty = global_node_page_state(NR_FILE_DIRTY);

                balance_domain_limits(gdtc, strictlimit);
                if (mdtc) {
                        /*
                         * If @wb belongs to !root memcg, repeat the same
                         * basic calculations for the memcg domain.
                         */
                        balance_domain_limits(mdtc, strictlimit);
                }

                if (!writeback_in_progress(wb) &&
                    (nr_dirty > gdtc->bg_thresh ||
                     (strictlimit && gdtc->wb_dirty > gdtc->wb_bg_thresh)))
                        wb_start_background_writeback(wb);

                /*
                 * If memcg domain is in effect, @dirty should be under
                 * both global and memcg freerun ceilings.
                 */
                if (gdtc->freerun && (!mdtc || mdtc->freerun)) {
                        unsigned long intv;
                        unsigned long m_intv;

free_running:
                        intv = domain_poll_intv(gdtc, strictlimit);
                        m_intv = ULONG_MAX;

                        current->dirty_paused_when = now;
                        current->nr_dirtied = 0;
                        if (mdtc)
                                m_intv = domain_poll_intv(mdtc, strictlimit);
                        current->nr_dirtied_pause = min(intv, m_intv);
                        break;
                }

                /*
                 * Unconditionally start background writeback if it's not
                 * already in progress. We need to do this because the global
                 * dirty threshold check above (nr_dirty > gdtc->bg_thresh)
                 * doesn't account for the memcg-based throttling case. memcg
                 * uses its own dirty count and thresholds and can trigger
                 * throttling even when global nr_dirty < gdtc->bg_thresh
                 *
                 * Writeback needs to be started else the writer stalls in the
                 * throttle loop waiting for dirty pages to be written back
                 * while no writeback is running.
                 */
                if (unlikely(!writeback_in_progress(wb)))
                        wb_start_background_writeback(wb);

                mem_cgroup_flush_foreign(wb);

                /*
                 * Calculate global domain's pos_ratio and select the
                 * global dtc by default.
                 */
                balance_wb_limits(gdtc, strictlimit);
                if (gdtc->freerun)
                        goto free_running;
                sdtc = gdtc;

                if (mdtc) {
                        /*
                         * If memcg domain is in effect, calculate its
                         * pos_ratio.  @wb should satisfy constraints from
                         * both global and memcg domains.  Choose the one
                         * w/ lower pos_ratio.
                         */
                        balance_wb_limits(mdtc, strictlimit);
                        if (mdtc->freerun)
                                goto free_running;
                        if (mdtc->pos_ratio < gdtc->pos_ratio)
                                sdtc = mdtc;
                }

                wb->dirty_exceeded = gdtc->dirty_exceeded ||
                                     (mdtc && mdtc->dirty_exceeded);
                if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +
                                           BANDWIDTH_INTERVAL))
                        __wb_update_bandwidth(gdtc, mdtc, true);

                /* throttle according to the chosen dtc */
                dirty_ratelimit = READ_ONCE(wb->dirty_ratelimit);
                task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >>
                                                        RATELIMIT_CALC_SHIFT;
                max_pause = wb_max_pause(wb, sdtc->wb_dirty);
                min_pause = wb_min_pause(wb, max_pause,
                                         task_ratelimit, dirty_ratelimit,
                                         &nr_dirtied_pause);

                if (unlikely(task_ratelimit == 0)) {
                        period = max_pause;
                        pause = max_pause;
                        goto pause;
                }
                period = HZ * pages_dirtied / task_ratelimit;
                pause = period;
                if (current->dirty_paused_when)
                        pause -= now - current->dirty_paused_when;
                /*
                 * For less than 1s think time (ext3/4 may block the dirtier
                 * for up to 800ms from time to time on 1-HDD; so does xfs,
                 * however at much less frequency), try to compensate it in
                 * future periods by updating the virtual time; otherwise just
                 * do a reset, as it may be a light dirtier.
                 */
                if (pause < min_pause) {
                        trace_balance_dirty_pages(wb,
                                                  sdtc,
                                                  dirty_ratelimit,
                                                  task_ratelimit,
                                                  pages_dirtied,
                                                  period,
                                                  min(pause, 0L),
                                                  start_time);
                        if (pause < -HZ) {
                                current->dirty_paused_when = now;
                                current->nr_dirtied = 0;
                        } else if (period) {
                                current->dirty_paused_when += period;
                                current->nr_dirtied = 0;
                        } else if (current->nr_dirtied_pause <= pages_dirtied)
                                current->nr_dirtied_pause += pages_dirtied;
                        break;
                }
                if (unlikely(pause > max_pause)) {
                        /* for occasional dropped task_ratelimit */
                        now += min(pause - max_pause, max_pause);
                        pause = max_pause;
                }

pause:
                trace_balance_dirty_pages(wb,
                                          sdtc,
                                          dirty_ratelimit,
                                          task_ratelimit,
                                          pages_dirtied,
                                          period,
                                          pause,
                                          start_time);
                if (flags & BDP_ASYNC) {
                        ret = -EAGAIN;
                        break;
                }
                __set_current_state(TASK_KILLABLE);
                bdi->last_bdp_sleep = jiffies;
                io_schedule_timeout(pause);

                current->dirty_paused_when = now + pause;
                current->nr_dirtied = 0;
                current->nr_dirtied_pause = nr_dirtied_pause;

                /*
                 * This is typically equal to (dirty < thresh) and can also
                 * keep "1000+ dd on a slow USB stick" under control.
                 */
                if (task_ratelimit)
                        break;

                /*
                 * In the case of an unresponsive NFS server and the NFS dirty
                 * pages exceeds dirty_thresh, give the other good wb's a pipe
                 * to go through, so that tasks on them still remain responsive.
                 *
                 * In theory 1 page is enough to keep the consumer-producer
                 * pipe going: the flusher cleans 1 page => the task dirties 1
                 * more page. However wb_dirty has accounting errors.  So use
                 * the larger and more IO friendly wb_stat_error.
                 */
                if (sdtc->wb_dirty <= wb_stat_error())
                        break;

                if (fatal_signal_pending(current))
                        break;
        }
        return ret;
}

static DEFINE_PER_CPU(int, bdp_ratelimits);

/*
 * Normal tasks are throttled by
 *        loop {
 *                dirty tsk->nr_dirtied_pause pages;
 *                take a snap in balance_dirty_pages();
 *        }
 * However there is a worst case. If every task exit immediately when dirtied
 * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be
 * called to throttle the page dirties. The solution is to save the not yet
 * throttled page dirties in dirty_throttle_leaks on task exit and charge them
 * randomly into the running tasks. This works well for the above worst case,
 * as the new task will pick up and accumulate the old task's leaked dirty
 * count and eventually get throttled.
 */
DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0;

/**
 * balance_dirty_pages_ratelimited_flags - Balance dirty memory state.
 * @mapping: address_space which was dirtied.
 * @flags: BDP flags.
 *
 * Processes which are dirtying memory should call in here once for each page
 * which was newly dirtied.  The function will periodically check the system's
 * dirty state and will initiate writeback if needed.
 *
 * See balance_dirty_pages_ratelimited() for details.
 *
 * Return: If @flags contains BDP_ASYNC, it may return -EAGAIN to
 * indicate that memory is out of balance and the caller must wait
 * for I/O to complete.  Otherwise, it will return 0 to indicate
 * that either memory was already in balance, or it was able to sleep
 * until the amount of dirty memory returned to balance.
 */
int balance_dirty_pages_ratelimited_flags(struct address_space *mapping,
                                        unsigned int flags)
{
        struct inode *inode = mapping->host;
        struct backing_dev_info *bdi = inode_to_bdi(inode);
        struct bdi_writeback *wb = NULL;
        int ratelimit;
        int ret = 0;
        int *p;

        if (!(bdi->capabilities & BDI_CAP_WRITEBACK))
                return ret;

        if (inode_cgwb_enabled(inode))
                wb = wb_get_create_current(bdi, GFP_KERNEL);
        if (!wb)
                wb = &bdi->wb;

        ratelimit = current->nr_dirtied_pause;
        if (wb->dirty_exceeded)
                ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10));

        preempt_disable();
        /*
         * This prevents one CPU to accumulate too many dirtied pages without
         * calling into balance_dirty_pages(), which can happen when there are
         * 1000+ tasks, all of them start dirtying pages at exactly the same
         * time, hence all honoured too large initial task->nr_dirtied_pause.
         */
        p =  this_cpu_ptr(&bdp_ratelimits);
        if (unlikely(current->nr_dirtied >= ratelimit))
                *p = 0;
        else if (unlikely(*p >= ratelimit_pages)) {
                *p = 0;
                ratelimit = 0;
        }
        /*
         * Pick up the dirtied pages by the exited tasks. This avoids lots of
         * short-lived tasks (eg. gcc invocations in a kernel build) escaping
         * the dirty throttling and livelock other long-run dirtiers.
         */
        p = this_cpu_ptr(&dirty_throttle_leaks);
        if (*p > 0 && current->nr_dirtied < ratelimit) {
                unsigned long nr_pages_dirtied;
                nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied);
                *p -= nr_pages_dirtied;
                current->nr_dirtied += nr_pages_dirtied;
        }
        preempt_enable();

        if (unlikely(current->nr_dirtied >= ratelimit))
                ret = balance_dirty_pages(wb, current->nr_dirtied, flags);

        wb_put(wb);
        return ret;
}
EXPORT_SYMBOL_GPL(balance_dirty_pages_ratelimited_flags);

/**
 * balance_dirty_pages_ratelimited - balance dirty memory state.
 * @mapping: address_space which was dirtied.
 *
 * Processes which are dirtying memory should call in here once for each page
 * which was newly dirtied.  The function will periodically check the system's
 * dirty state and will initiate writeback if needed.
 *
 * Once we're over the dirty memory limit we decrease the ratelimiting
 * by a lot, to prevent individual processes from overshooting the limit
 * by (ratelimit_pages) each.
 */
void balance_dirty_pages_ratelimited(struct address_space *mapping)
{
        balance_dirty_pages_ratelimited_flags(mapping, 0);
}
EXPORT_SYMBOL(balance_dirty_pages_ratelimited);

/*
 * Similar to wb_dirty_limits, wb_bg_dirty_limits also calculates dirty
 * and thresh, but it's for background writeback.
 */
static void wb_bg_dirty_limits(struct dirty_throttle_control *dtc)
{
        struct bdi_writeback *wb = dtc->wb;

        dtc->wb_bg_thresh = __wb_calc_thresh(dtc, dtc->bg_thresh);
        if (dtc->wb_bg_thresh < 2 * wb_stat_error())
                dtc->wb_dirty = wb_stat_sum(wb, WB_RECLAIMABLE);
        else
                dtc->wb_dirty = wb_stat(wb, WB_RECLAIMABLE);
}

static bool domain_over_bg_thresh(struct dirty_throttle_control *dtc)
{
        domain_dirty_avail(dtc, false);
        domain_dirty_limits(dtc);
        if (dtc->dirty > dtc->bg_thresh)
                return true;

        wb_bg_dirty_limits(dtc);
        if (dtc->wb_dirty > dtc->wb_bg_thresh)
                return true;

        return false;
}

/**
 * wb_over_bg_thresh - does @wb need to be written back?
 * @wb: bdi_writeback of interest
 *
 * Determines whether background writeback should keep writing @wb or it's
 * clean enough.
 *
 * Return: %true if writeback should continue.
 */
bool wb_over_bg_thresh(struct bdi_writeback *wb)
{
        struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };
        struct dirty_throttle_control mdtc = { MDTC_INIT(wb, &gdtc) };

        if (domain_over_bg_thresh(&gdtc))
                return true;

        if (mdtc_valid(&mdtc))
                return domain_over_bg_thresh(&mdtc);

        return false;
}

#ifdef CONFIG_SYSCTL
/*
 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
 */
static int dirty_writeback_centisecs_handler(const struct ctl_table *table, int write,
                void *buffer, size_t *length, loff_t *ppos)
{
        unsigned int old_interval = dirty_writeback_interval;
        int ret;

        ret = proc_dointvec(table, write, buffer, length, ppos);

        /*
         * Writing 0 to dirty_writeback_interval will disable periodic writeback
         * and a different non-zero value will wakeup the writeback threads.
         * wb_wakeup_delayed() would be more appropriate, but it's a pain to
         * iterate over all bdis and wbs.
         * The reason we do this is to make the change take effect immediately.
         */
        if (!ret && write && dirty_writeback_interval &&
                dirty_writeback_interval != old_interval)
                wakeup_flusher_threads(WB_REASON_PERIODIC);

        return ret;
}
#endif

/*
 * If ratelimit_pages is too high then we can get into dirty-data overload
 * if a large number of processes all perform writes at the same time.
 *
 * Here we set ratelimit_pages to a level which ensures that when all CPUs are
 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory
 * thresholds.
 */

void writeback_set_ratelimit(void)
{
        struct wb_domain *dom = &global_wb_domain;
        unsigned long background_thresh;
        unsigned long dirty_thresh;

        global_dirty_limits(&background_thresh, &dirty_thresh);
        dom->dirty_limit = dirty_thresh;
        ratelimit_pages = dirty_thresh / (num_online_cpus() * 32);
        if (ratelimit_pages < 16)
                ratelimit_pages = 16;
}

static int page_writeback_cpu_online(unsigned int cpu)
{
        writeback_set_ratelimit();
        return 0;
}

#ifdef CONFIG_SYSCTL

static int laptop_mode;
static int laptop_mode_handler(const struct ctl_table *table, int write,
                               void *buffer, size_t *lenp, loff_t *ppos)
{
        int ret = proc_dointvec_jiffies(table, write, buffer, lenp, ppos);

        if (!ret && write)
                pr_warn("%s: vm.laptop_mode is deprecated. Ignoring setting.\n",
                        current->comm);

        return ret;
}

/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */
static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE;

static const struct ctl_table vm_page_writeback_sysctls[] = {
        {
                .procname   = "dirty_background_ratio",
                .data       = &dirty_background_ratio,
                .maxlen     = sizeof(dirty_background_ratio),
                .mode       = 0644,
                .proc_handler   = dirty_background_ratio_handler,
                .extra1     = SYSCTL_ZERO,
                .extra2     = SYSCTL_ONE_HUNDRED,
        },
        {
                .procname   = "dirty_background_bytes",
                .data       = &dirty_background_bytes,
                .maxlen     = sizeof(dirty_background_bytes),
                .mode       = 0644,
                .proc_handler   = dirty_background_bytes_handler,
                .extra1     = SYSCTL_LONG_ONE,
        },
        {
                .procname   = "dirty_ratio",
                .data       = &vm_dirty_ratio,
                .maxlen     = sizeof(vm_dirty_ratio),
                .mode       = 0644,
                .proc_handler   = dirty_ratio_handler,
                .extra1     = SYSCTL_ZERO,
                .extra2     = SYSCTL_ONE_HUNDRED,
        },
        {
                .procname   = "dirty_bytes",
                .data       = &vm_dirty_bytes,
                .maxlen     = sizeof(vm_dirty_bytes),
                .mode       = 0644,
                .proc_handler   = dirty_bytes_handler,
                .extra1     = (void *)&dirty_bytes_min,
        },
        {
                .procname   = "dirty_writeback_centisecs",
                .data       = &dirty_writeback_interval,
                .maxlen     = sizeof(dirty_writeback_interval),
                .mode       = 0644,
                .proc_handler   = dirty_writeback_centisecs_handler,
        },
        {
                .procname   = "dirty_expire_centisecs",
                .data       = &dirty_expire_interval,
                .maxlen     = sizeof(dirty_expire_interval),
                .mode       = 0644,
                .proc_handler   = proc_dointvec_minmax,
                .extra1     = SYSCTL_ZERO,
        },
#ifdef CONFIG_HIGHMEM
        {
                .procname        = "highmem_is_dirtyable",
                .data                = &vm_highmem_is_dirtyable,
                .maxlen                = sizeof(vm_highmem_is_dirtyable),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
#endif
        {
                .procname        = "laptop_mode",
                .data                = &laptop_mode,
                .maxlen                = sizeof(laptop_mode),
                .mode                = 0644,
                .proc_handler        = laptop_mode_handler,
        },
};
#endif

/*
 * Called early on to tune the page writeback dirty limits.
 *
 * We used to scale dirty pages according to how total memory
 * related to pages that could be allocated for buffers.
 *
 * However, that was when we used "dirty_ratio" to scale with
 * all memory, and we don't do that any more. "dirty_ratio"
 * is now applied to total non-HIGHPAGE memory, and as such we can't
 * get into the old insane situation any more where we had
 * large amounts of dirty pages compared to a small amount of
 * non-HIGHMEM memory.
 *
 * But we might still want to scale the dirty_ratio by how
 * much memory the box has..
 */
void __init page_writeback_init(void)
{
        BUG_ON(wb_domain_init(&global_wb_domain, GFP_KERNEL));

        cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mm/writeback:online",
                          page_writeback_cpu_online, NULL);
        cpuhp_setup_state(CPUHP_MM_WRITEBACK_DEAD, "mm/writeback:dead", NULL,
                          page_writeback_cpu_online);
#ifdef CONFIG_SYSCTL
        register_sysctl_init("vm", vm_page_writeback_sysctls);
#endif
}

/**
 * tag_pages_for_writeback - tag pages to be written by writeback
 * @mapping: address space structure to write
 * @start: starting page index
 * @end: ending page index (inclusive)
 *
 * This function scans the page range from @start to @end (inclusive) and tags
 * all pages that have DIRTY tag set with a special TOWRITE tag.  The caller
 * can then use the TOWRITE tag to identify pages eligible for writeback.
 * This mechanism is used to avoid livelocking of writeback by a process
 * steadily creating new dirty pages in the file (thus it is important for this
 * function to be quick so that it can tag pages faster than a dirtying process
 * can create them).
 */
void tag_pages_for_writeback(struct address_space *mapping,
                             pgoff_t start, pgoff_t end)
{
        XA_STATE(xas, &mapping->i_pages, start);
        unsigned int tagged = 0;
        void *page;

        xas_lock_irq(&xas);
        xas_for_each_marked(&xas, page, end, PAGECACHE_TAG_DIRTY) {
                xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE);
                if (++tagged % XA_CHECK_SCHED)
                        continue;

                xas_pause(&xas);
                xas_unlock_irq(&xas);
                cond_resched();
                xas_lock_irq(&xas);
        }
        xas_unlock_irq(&xas);
}
EXPORT_SYMBOL(tag_pages_for_writeback);

static bool folio_prepare_writeback(struct address_space *mapping,
                struct writeback_control *wbc, struct folio *folio)
{
        /*
         * Folio truncated or invalidated. We can freely skip it then,
         * even for data integrity operations: the folio has disappeared
         * concurrently, so there could be no real expectation of this
         * data integrity operation even if there is now a new, dirty
         * folio at the same pagecache index.
         */
        if (unlikely(folio->mapping != mapping))
                return false;

        /*
         * Did somebody else write it for us?
         */
        if (!folio_test_dirty(folio))
                return false;

        if (folio_test_writeback(folio)) {
                if (wbc->sync_mode == WB_SYNC_NONE)
                        return false;
                folio_wait_writeback(folio);
        }
        BUG_ON(folio_test_writeback(folio));

        if (!folio_clear_dirty_for_io(folio))
                return false;

        return true;
}


static pgoff_t wbc_end(struct writeback_control *wbc)
{
        if (wbc->range_cyclic)
                return -1;
        return wbc->range_end >> PAGE_SHIFT;
}

static struct folio *writeback_get_folio(struct address_space *mapping,
                struct writeback_control *wbc)
{
        struct folio *folio;

retry:
        folio = folio_batch_next(&wbc->fbatch);
        if (!folio) {
                folio_batch_release(&wbc->fbatch);
                cond_resched();
                filemap_get_folios_tag(mapping, &wbc->index, wbc_end(wbc),
                                wbc_to_tag(wbc), &wbc->fbatch);
                folio = folio_batch_next(&wbc->fbatch);
                if (!folio)
                        return NULL;
        }

        folio_lock(folio);
        if (unlikely(!folio_prepare_writeback(mapping, wbc, folio))) {
                folio_unlock(folio);
                goto retry;
        }

        trace_wbc_writepage(wbc, inode_to_bdi(mapping->host));
        return folio;
}

/**
 * writeback_iter - iterate folio of a mapping for writeback
 * @mapping: address space structure to write
 * @wbc: writeback context
 * @folio: previously iterated folio (%NULL to start)
 * @error: in-out pointer for writeback errors (see below)
 *
 * This function returns the next folio for the writeback operation described by
 * @wbc on @mapping and  should be called in a while loop in the ->writepages
 * implementation.
 *
 * To start the writeback operation, %NULL is passed in the @folio argument, and
 * for every subsequent iteration the folio returned previously should be passed
 * back in.
 *
 * If there was an error in the per-folio writeback inside the writeback_iter()
 * loop, @error should be set to the error value.
 *
 * Once the writeback described in @wbc has finished, this function will return
 * %NULL and if there was an error in any iteration restore it to @error.
 *
 * Note: callers should not manually break out of the loop using break or goto
 * but must keep calling writeback_iter() until it returns %NULL.
 *
 * Return: the folio to write or %NULL if the loop is done.
 */
struct folio *writeback_iter(struct address_space *mapping,
                struct writeback_control *wbc, struct folio *folio, int *error)
{
        if (!folio) {
                folio_batch_init(&wbc->fbatch);
                wbc->saved_err = *error = 0;

                /*
                 * For range cyclic writeback we remember where we stopped so
                 * that we can continue where we stopped.
                 *
                 * For non-cyclic writeback we always start at the beginning of
                 * the passed in range.
                 */
                if (wbc->range_cyclic)
                        wbc->index = mapping->writeback_index;
                else
                        wbc->index = wbc->range_start >> PAGE_SHIFT;

                /*
                 * To avoid livelocks when other processes dirty new pages, we
                 * first tag pages which should be written back and only then
                 * start writing them.
                 *
                 * For data-integrity writeback we have to be careful so that we
                 * do not miss some pages (e.g., because some other process has
                 * cleared the TOWRITE tag we set).  The rule we follow is that
                 * TOWRITE tag can be cleared only by the process clearing the
                 * DIRTY tag (and submitting the page for I/O).
                 */
                if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
                        tag_pages_for_writeback(mapping, wbc->index,
                                        wbc_end(wbc));
        } else {
                wbc->nr_to_write -= folio_nr_pages(folio);

                WARN_ON_ONCE(*error > 0);

                /*
                 * For integrity writeback we have to keep going until we have
                 * written all the folios we tagged for writeback above, even if
                 * we run past wbc->nr_to_write or encounter errors.
                 * We stash away the first error we encounter in wbc->saved_err
                 * so that it can be retrieved when we're done.  This is because
                 * the file system may still have state to clear for each folio.
                 *
                 * For background writeback we exit as soon as we run past
                 * wbc->nr_to_write or encounter the first error.
                 */
                if (wbc->sync_mode == WB_SYNC_ALL) {
                        if (*error && !wbc->saved_err)
                                wbc->saved_err = *error;
                } else {
                        if (*error || wbc->nr_to_write <= 0)
                                goto done;
                }
        }

        folio = writeback_get_folio(mapping, wbc);
        if (!folio) {
                /*
                 * To avoid deadlocks between range_cyclic writeback and callers
                 * that hold folios in writeback to aggregate I/O until
                 * the writeback iteration finishes, we do not loop back to the
                 * start of the file.  Doing so causes a folio lock/folio
                 * writeback access order inversion - we should only ever lock
                 * multiple folios in ascending folio->index order, and looping
                 * back to the start of the file violates that rule and causes
                 * deadlocks.
                 */
                if (wbc->range_cyclic)
                        mapping->writeback_index = 0;

                /*
                 * Return the first error we encountered (if there was any) to
                 * the caller.
                 */
                *error = wbc->saved_err;
        }
        return folio;

done:
        if (wbc->range_cyclic)
                mapping->writeback_index = folio_next_index(folio);
        folio_batch_release(&wbc->fbatch);
        return NULL;
}
EXPORT_SYMBOL_GPL(writeback_iter);

int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
{
        int ret;
        struct bdi_writeback *wb;

        if (wbc->nr_to_write <= 0)
                return 0;
        wb = inode_to_wb_wbc(mapping->host, wbc);
        wb_bandwidth_estimate_start(wb);
        while (1) {
                if (mapping->a_ops->writepages)
                        ret = mapping->a_ops->writepages(mapping, wbc);
                else
                        /* deal with chardevs and other special files */
                        ret = 0;
                if (ret != -ENOMEM || wbc->sync_mode != WB_SYNC_ALL)
                        break;

                /*
                 * Lacking an allocation context or the locality or writeback
                 * state of any of the inode's pages, throttle based on
                 * writeback activity on the local node. It's as good a
                 * guess as any.
                 */
                reclaim_throttle(NODE_DATA(numa_node_id()),
                        VMSCAN_THROTTLE_WRITEBACK);
        }
        /*
         * Usually few pages are written by now from those we've just submitted
         * but if there's constant writeback being submitted, this makes sure
         * writeback bandwidth is updated once in a while.
         */
        if (time_is_before_jiffies(READ_ONCE(wb->bw_time_stamp) +
                                   BANDWIDTH_INTERVAL))
                wb_update_bandwidth(wb);
        return ret;
}

/*
 * For address_spaces which do not use buffers nor write back.
 */
bool noop_dirty_folio(struct address_space *mapping, struct folio *folio)
{
        if (!folio_test_dirty(folio))
                return !folio_test_set_dirty(folio);
        return false;
}
EXPORT_SYMBOL(noop_dirty_folio);

/*
 * Helper function for set_page_dirty family.
 *
 * NOTE: This relies on being atomic wrt interrupts.
 */
static void folio_account_dirtied(struct folio *folio,
                struct address_space *mapping)
{
        struct inode *inode = mapping->host;

        trace_writeback_dirty_folio(folio, mapping);

        if (mapping_can_writeback(mapping)) {
                struct bdi_writeback *wb;
                long nr = folio_nr_pages(folio);

                inode_attach_wb(inode, folio);
                wb = inode_to_wb(inode);

                lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, nr);
                __zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, nr);
                __node_stat_mod_folio(folio, NR_DIRTIED, nr);
                wb_stat_mod(wb, WB_RECLAIMABLE, nr);
                wb_stat_mod(wb, WB_DIRTIED, nr);
                task_io_account_write(nr * PAGE_SIZE);
                current->nr_dirtied += nr;
                __this_cpu_add(bdp_ratelimits, nr);

                mem_cgroup_track_foreign_dirty(folio, wb);
        }
}

/*
 * Helper function for deaccounting dirty page without writeback.
 *
 */
void folio_account_cleaned(struct folio *folio, struct bdi_writeback *wb)
{
        long nr = folio_nr_pages(folio);

        lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, -nr);
        zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr);
        wb_stat_mod(wb, WB_RECLAIMABLE, -nr);
        task_io_account_cancelled_write(nr * PAGE_SIZE);
}

/*
 * Mark the folio dirty, and set it dirty in the page cache.
 *
 * If warn is true, then emit a warning if the folio is not uptodate and has
 * not been truncated.
 *
 * It is the caller's responsibility to prevent the folio from being truncated
 * while this function is in progress, although it may have been truncated
 * before this function is called.  Most callers have the folio locked.
 * A few have the folio blocked from truncation through other means (e.g.
 * zap_vma() has it mapped and is holding the page table lock).
 * When called from mark_buffer_dirty(), the filesystem should hold a
 * reference to the buffer_head that is being marked dirty, which causes
 * try_to_free_buffers() to fail.
 */
void __folio_mark_dirty(struct folio *folio, struct address_space *mapping,
                             int warn)
{
        unsigned long flags;

        /*
         * Shmem writeback relies on swap, and swap writeback is LRU based,
         * not using the dirty mark.
         */
        VM_WARN_ON_ONCE(folio_test_swapcache(folio) || shmem_mapping(mapping));

        xa_lock_irqsave(&mapping->i_pages, flags);
        if (folio->mapping) {        /* Race with truncate? */
                WARN_ON_ONCE(warn && !folio_test_uptodate(folio));
                folio_account_dirtied(folio, mapping);
                __xa_set_mark(&mapping->i_pages, folio->index,
                              PAGECACHE_TAG_DIRTY);
        }
        xa_unlock_irqrestore(&mapping->i_pages, flags);
}

/**
 * filemap_dirty_folio - Mark a folio dirty for filesystems which do not use buffer_heads.
 * @mapping: Address space this folio belongs to.
 * @folio: Folio to be marked as dirty.
 *
 * Filesystems which do not use buffer heads should call this function
 * from their dirty_folio address space operation.  It ignores the
 * contents of folio_get_private(), so if the filesystem marks individual
 * blocks as dirty, the filesystem should handle that itself.
 *
 * This is also sometimes used by filesystems which use buffer_heads when
 * a single buffer is being dirtied: we want to set the folio dirty in
 * that case, but not all the buffers.  This is a "bottom-up" dirtying,
 * whereas block_dirty_folio() is a "top-down" dirtying.
 *
 * The caller must ensure this doesn't race with truncation.  Most will
 * simply hold the folio lock, but e.g. zap_pte_range() calls with the
 * folio mapped and the pte lock held, which also locks out truncation.
 */
bool filemap_dirty_folio(struct address_space *mapping, struct folio *folio)
{
        if (folio_test_set_dirty(folio))
                return false;

        __folio_mark_dirty(folio, mapping, !folio_test_private(folio));

        if (mapping->host) {
                /* !PageAnon && !swapper_space */
                __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
        }
        return true;
}
EXPORT_SYMBOL(filemap_dirty_folio);

/**
 * folio_redirty_for_writepage - Decline to write a dirty folio.
 * @wbc: The writeback control.
 * @folio: The folio.
 *
 * When a writepage implementation decides that it doesn't want to write
 * @folio for some reason, it should call this function, unlock @folio and
 * return 0.
 *
 * Return: True if we redirtied the folio.  False if someone else dirtied
 * it first.
 */
bool folio_redirty_for_writepage(struct writeback_control *wbc,
                struct folio *folio)
{
        struct address_space *mapping = folio->mapping;
        long nr = folio_nr_pages(folio);
        bool ret;

        wbc->pages_skipped += nr;
        ret = filemap_dirty_folio(mapping, folio);
        if (mapping && mapping_can_writeback(mapping)) {
                struct inode *inode = mapping->host;
                struct bdi_writeback *wb;
                struct wb_lock_cookie cookie = {};

                wb = unlocked_inode_to_wb_begin(inode, &cookie);
                current->nr_dirtied -= nr;
                node_stat_mod_folio(folio, NR_DIRTIED, -nr);
                wb_stat_mod(wb, WB_DIRTIED, -nr);
                unlocked_inode_to_wb_end(inode, &cookie);
        }
        return ret;
}
EXPORT_SYMBOL(folio_redirty_for_writepage);

/**
 * folio_mark_dirty - Mark a folio as being modified.
 * @folio: The folio.
 *
 * The folio may not be truncated while this function is running.
 * Holding the folio lock is sufficient to prevent truncation, but some
 * callers cannot acquire a sleeping lock.  These callers instead hold
 * the page table lock for a page table which contains at least one page
 * in this folio.  Truncation will block on the page table lock as it
 * unmaps pages before removing the folio from its mapping.
 *
 * Return: True if the folio was newly dirtied, false if it was already dirty.
 */
bool folio_mark_dirty(struct folio *folio)
{
        struct address_space *mapping = folio_mapping(folio);

        if (likely(mapping)) {
                /*
                 * readahead/folio_deactivate could remain
                 * PG_readahead/PG_reclaim due to race with folio_end_writeback
                 * About readahead, if the folio is written, the flags would be
                 * reset. So no problem.
                 * About folio_deactivate, if the folio is redirtied,
                 * the flag will be reset. So no problem. but if the
                 * folio is used by readahead it will confuse readahead
                 * and make it restart the size rampup process. But it's
                 * a trivial problem.
                 */
                if (folio_test_reclaim(folio))
                        folio_clear_reclaim(folio);
                return mapping->a_ops->dirty_folio(mapping, folio);
        }

        return noop_dirty_folio(mapping, folio);
}
EXPORT_SYMBOL(folio_mark_dirty);

/*
 * folio_mark_dirty() is racy if the caller has no reference against
 * folio->mapping->host, and if the folio is unlocked.  This is because another
 * CPU could truncate the folio off the mapping and then free the mapping.
 *
 * Usually, the folio _is_ locked, or the caller is a user-space process which
 * holds a reference on the inode by having an open file.
 *
 * In other cases, the folio should be locked before running folio_mark_dirty().
 */
bool folio_mark_dirty_lock(struct folio *folio)
{
        bool ret;

        folio_lock(folio);
        ret = folio_mark_dirty(folio);
        folio_unlock(folio);
        return ret;
}
EXPORT_SYMBOL(folio_mark_dirty_lock);

/*
 * This cancels just the dirty bit on the kernel page itself, it does NOT
 * actually remove dirty bits on any mmap's that may be around. It also
 * leaves the page tagged dirty, so any sync activity will still find it on
 * the dirty lists, and in particular, clear_page_dirty_for_io() will still
 * look at the dirty bits in the VM.
 *
 * Doing this should *normally* only ever be done when a page is truncated,
 * and is not actually mapped anywhere at all. However, fs/buffer.c does
 * this when it notices that somebody has cleaned out all the buffers on a
 * page without actually doing it through the VM. Can you say "ext3 is
 * horribly ugly"? Thought you could.
 */
void __folio_cancel_dirty(struct folio *folio)
{
        struct address_space *mapping = folio_mapping(folio);

        if (mapping_can_writeback(mapping)) {
                struct inode *inode = mapping->host;
                struct bdi_writeback *wb;
                struct wb_lock_cookie cookie = {};

                wb = unlocked_inode_to_wb_begin(inode, &cookie);

                if (folio_test_clear_dirty(folio))
                        folio_account_cleaned(folio, wb);

                unlocked_inode_to_wb_end(inode, &cookie);
        } else {
                folio_clear_dirty(folio);
        }
}
EXPORT_SYMBOL(__folio_cancel_dirty);

/*
 * Clear a folio's dirty flag, while caring for dirty memory accounting.
 * Returns true if the folio was previously dirty.
 *
 * This is for preparing to put the folio under writeout.  We leave
 * the folio tagged as dirty in the xarray so that a concurrent
 * write-for-sync can discover it via a PAGECACHE_TAG_DIRTY walk.
 * The ->writepage implementation will run either folio_start_writeback()
 * or folio_mark_dirty(), at which stage we bring the folio's dirty flag
 * and xarray dirty tag back into sync.
 *
 * This incoherency between the folio's dirty flag and xarray tag is
 * unfortunate, but it only exists while the folio is locked.
 */
bool folio_clear_dirty_for_io(struct folio *folio)
{
        struct address_space *mapping = folio_mapping(folio);
        bool ret = false;

        VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);

        if (mapping && mapping_can_writeback(mapping)) {
                struct inode *inode = mapping->host;
                struct bdi_writeback *wb;
                struct wb_lock_cookie cookie = {};

                /*
                 * Yes, Virginia, this is indeed insane.
                 *
                 * We use this sequence to make sure that
                 *  (a) we account for dirty stats properly
                 *  (b) we tell the low-level filesystem to
                 *      mark the whole folio dirty if it was
                 *      dirty in a pagetable. Only to then
                 *  (c) clean the folio again and return 1 to
                 *      cause the writeback.
                 *
                 * This way we avoid all nasty races with the
                 * dirty bit in multiple places and clearing
                 * them concurrently from different threads.
                 *
                 * Note! Normally the "folio_mark_dirty(folio)"
                 * has no effect on the actual dirty bit - since
                 * that will already usually be set. But we
                 * need the side effects, and it can help us
                 * avoid races.
                 *
                 * We basically use the folio "master dirty bit"
                 * as a serialization point for all the different
                 * threads doing their things.
                 */
                if (folio_mkclean(folio))
                        folio_mark_dirty(folio);
                /*
                 * We carefully synchronise fault handlers against
                 * installing a dirty pte and marking the folio dirty
                 * at this point.  We do this by having them hold the
                 * page lock while dirtying the folio, and folios are
                 * always locked coming in here, so we get the desired
                 * exclusion.
                 */
                wb = unlocked_inode_to_wb_begin(inode, &cookie);
                if (folio_test_clear_dirty(folio)) {
                        long nr = folio_nr_pages(folio);
                        lruvec_stat_mod_folio(folio, NR_FILE_DIRTY, -nr);
                        zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr);
                        wb_stat_mod(wb, WB_RECLAIMABLE, -nr);
                        ret = true;
                }
                unlocked_inode_to_wb_end(inode, &cookie);
                return ret;
        }
        return folio_test_clear_dirty(folio);
}
EXPORT_SYMBOL(folio_clear_dirty_for_io);

static void wb_inode_writeback_start(struct bdi_writeback *wb)
{
        atomic_inc(&wb->writeback_inodes);
}

static void wb_inode_writeback_end(struct bdi_writeback *wb)
{
        unsigned long flags;
        atomic_dec(&wb->writeback_inodes);
        /*
         * Make sure estimate of writeback throughput gets updated after
         * writeback completed. We delay the update by BANDWIDTH_INTERVAL
         * (which is the interval other bandwidth updates use for batching) so
         * that if multiple inodes end writeback at a similar time, they get
         * batched into one bandwidth update.
         */
        spin_lock_irqsave(&wb->work_lock, flags);
        if (test_bit(WB_registered, &wb->state))
                queue_delayed_work(bdi_wq, &wb->bw_dwork, BANDWIDTH_INTERVAL);
        spin_unlock_irqrestore(&wb->work_lock, flags);
}

bool __folio_end_writeback(struct folio *folio)
{
        long nr = folio_nr_pages(folio);
        struct address_space *mapping = folio_mapping(folio);
        bool ret;

        if (mapping && mapping_use_writeback_tags(mapping)) {
                struct inode *inode = mapping->host;
                struct bdi_writeback *wb;
                unsigned long flags;

                xa_lock_irqsave(&mapping->i_pages, flags);
                ret = folio_xor_flags_has_waiters(folio, 1 << PG_writeback);
                __xa_clear_mark(&mapping->i_pages, folio->index,
                                        PAGECACHE_TAG_WRITEBACK);

                wb = inode_to_wb(inode);
                wb_stat_mod(wb, WB_WRITEBACK, -nr);
                __wb_writeout_add(wb, nr);
                if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) {
                        wb_inode_writeback_end(wb);
                        if (mapping->host)
                                sb_clear_inode_writeback(mapping->host);
                }

                xa_unlock_irqrestore(&mapping->i_pages, flags);
        } else {
                ret = folio_xor_flags_has_waiters(folio, 1 << PG_writeback);
        }

        lruvec_stat_mod_folio(folio, NR_WRITEBACK, -nr);
        zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, -nr);
        node_stat_mod_folio(folio, NR_WRITTEN, nr);

        return ret;
}

void __folio_start_writeback(struct folio *folio, bool keep_write)
{
        long nr = folio_nr_pages(folio);
        struct address_space *mapping = folio_mapping(folio);
        int access_ret;

        VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);
        VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);

        if (mapping && mapping_use_writeback_tags(mapping)) {
                XA_STATE(xas, &mapping->i_pages, folio->index);
                struct inode *inode = mapping->host;
                struct bdi_writeback *wb;
                unsigned long flags;
                bool on_wblist;

                xas_lock_irqsave(&xas, flags);
                xas_load(&xas);
                folio_test_set_writeback(folio);

                on_wblist = mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);

                xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
                wb = inode_to_wb(inode);
                wb_stat_mod(wb, WB_WRITEBACK, nr);
                if (!on_wblist) {
                        wb_inode_writeback_start(wb);
                        /*
                         * We can come through here when swapping anonymous
                         * folios, so we don't necessarily have an inode to
                         * track for sync.
                         */
                        if (mapping->host)
                                sb_mark_inode_writeback(mapping->host);
                }

                if (!folio_test_dirty(folio))
                        xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
                if (!keep_write)
                        xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
                xas_unlock_irqrestore(&xas, flags);
        } else {
                folio_test_set_writeback(folio);
        }

        lruvec_stat_mod_folio(folio, NR_WRITEBACK, nr);
        zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, nr);

        access_ret = arch_make_folio_accessible(folio);
        /*
         * If writeback has been triggered on a page that cannot be made
         * accessible, it is too late to recover here.
         */
        VM_BUG_ON_FOLIO(access_ret != 0, folio);
}
EXPORT_SYMBOL(__folio_start_writeback);

/**
 * folio_wait_writeback - Wait for a folio to finish writeback.
 * @folio: The folio to wait for.
 *
 * If the folio is currently being written back to storage, wait for the
 * I/O to complete.
 *
 * Context: Sleeps.  Must be called in process context and with
 * no spinlocks held.  Caller should hold a reference on the folio.
 * If the folio is not locked, writeback may start again after writeback
 * has finished.
 */
void folio_wait_writeback(struct folio *folio)
{
        while (folio_test_writeback(folio)) {
                trace_folio_wait_writeback(folio, folio_mapping(folio));
                folio_wait_bit(folio, PG_writeback);
        }
}
EXPORT_SYMBOL_GPL(folio_wait_writeback);

/**
 * folio_wait_writeback_killable - Wait for a folio to finish writeback.
 * @folio: The folio to wait for.
 *
 * If the folio is currently being written back to storage, wait for the
 * I/O to complete or a fatal signal to arrive.
 *
 * Context: Sleeps.  Must be called in process context and with
 * no spinlocks held.  Caller should hold a reference on the folio.
 * If the folio is not locked, writeback may start again after writeback
 * has finished.
 * Return: 0 on success, -EINTR if we get a fatal signal while waiting.
 */
int folio_wait_writeback_killable(struct folio *folio)
{
        while (folio_test_writeback(folio)) {
                trace_folio_wait_writeback(folio, folio_mapping(folio));
                if (folio_wait_bit_killable(folio, PG_writeback))
                        return -EINTR;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(folio_wait_writeback_killable);

/**
 * folio_wait_stable() - wait for writeback to finish, if necessary.
 * @folio: The folio to wait on.
 *
 * This function determines if the given folio is related to a backing
 * device that requires folio contents to be held stable during writeback.
 * If so, then it will wait for any pending writeback to complete.
 *
 * Context: Sleeps.  Must be called in process context and with
 * no spinlocks held.  Caller should hold a reference on the folio.
 * If the folio is not locked, writeback may start again after writeback
 * has finished.
 */
void folio_wait_stable(struct folio *folio)
{
        if (mapping_stable_writes(folio_mapping(folio)))
                folio_wait_writeback(folio);
}
EXPORT_SYMBOL_GPL(folio_wait_stable);















































































































































   20 





   20 
   20 
   20 




   20 












    3 










    3 






































































    2 








    2 























































































































































    9 




















    9 







    9 




























    7 







    7 






















































































































































































































































































































































































































































































































































































   11 









    9 





    9 












































































    2 






























































































    4 






    2 





    2 



    1 






































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
// SPDX-License-Identifier: GPL-2.0-or-later
/*****************************************************************************
 * Linux PPP over L2TP (PPPoX/PPPoL2TP) Sockets
 *
 * PPPoX    --- Generic PPP encapsulation socket family
 * PPPoL2TP --- PPP over L2TP (RFC 2661)
 *
 * Version:        2.0.0
 *
 * Authors:        James Chapman (jchapman@katalix.com)
 *
 * Based on original work by Martijn van Oosterhout <kleptog@svana.org>
 *
 * License:
 */

/* This driver handles only L2TP data frames; control frames are handled by a
 * userspace application.
 *
 * To send data in an L2TP session, userspace opens a PPPoL2TP socket and
 * attaches it to a bound UDP socket with local tunnel_id / session_id and
 * peer tunnel_id / session_id set. Data can then be sent or received using
 * regular socket sendmsg() / recvmsg() calls. Kernel parameters of the socket
 * can be read or modified using ioctl() or [gs]etsockopt() calls.
 *
 * When a PPPoL2TP socket is connected with local and peer session_id values
 * zero, the socket is treated as a special tunnel management socket.
 *
 * Here's example userspace code to create a socket for sending/receiving data
 * over an L2TP session:-
 *
 *        struct sockaddr_pppol2tp sax;
 *        int fd;
 *        int session_fd;
 *
 *        fd = socket(AF_PPPOX, SOCK_DGRAM, PX_PROTO_OL2TP);
 *
 *        sax.sa_family = AF_PPPOX;
 *        sax.sa_protocol = PX_PROTO_OL2TP;
 *        sax.pppol2tp.fd = tunnel_fd;        // bound UDP socket
 *        sax.pppol2tp.addr.sin_addr.s_addr = addr->sin_addr.s_addr;
 *        sax.pppol2tp.addr.sin_port = addr->sin_port;
 *        sax.pppol2tp.addr.sin_family = AF_INET;
 *        sax.pppol2tp.s_tunnel  = tunnel_id;
 *        sax.pppol2tp.s_session = session_id;
 *        sax.pppol2tp.d_tunnel  = peer_tunnel_id;
 *        sax.pppol2tp.d_session = peer_session_id;
 *
 *        session_fd = connect(fd, (struct sockaddr *)&sax, sizeof(sax));
 *
 * A pppd plugin that allows PPP traffic to be carried over L2TP using
 * this driver is available from the OpenL2TP project at
 * http://openl2tp.sourceforge.net.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/string.h>
#include <linux/list.h>
#include <linux/uaccess.h>

#include <linux/kernel.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/errno.h>
#include <linux/jiffies.h>

#include <linux/netdevice.h>
#include <linux/net.h>
#include <linux/inetdevice.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/ip.h>
#include <linux/udp.h>
#include <linux/if_pppox.h>
#include <linux/if_pppol2tp.h>
#include <net/sock.h>
#include <linux/ppp_channel.h>
#include <linux/ppp_defs.h>
#include <linux/ppp-ioctl.h>
#include <linux/file.h>
#include <linux/hash.h>
#include <linux/sort.h>
#include <linux/proc_fs.h>
#include <linux/l2tp.h>
#include <linux/nsproxy.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/ip.h>
#include <net/udp.h>
#include <net/inet_common.h>

#include <asm/byteorder.h>
#include <linux/atomic.h>

#include "l2tp_core.h"

#define PPPOL2TP_DRV_VERSION        "V2.0"

/* Space for UDP, L2TP and PPP headers */
#define PPPOL2TP_HEADER_OVERHEAD        40

/* Number of bytes to build transmit L2TP headers.
 * Unfortunately the size is different depending on whether sequence numbers
 * are enabled.
 */
#define PPPOL2TP_L2TP_HDR_SIZE_SEQ                10
#define PPPOL2TP_L2TP_HDR_SIZE_NOSEQ                6

/* Private data of each session. This data lives at the end of struct
 * l2tp_session, referenced via session->priv[].
 */
struct pppol2tp_session {
        int                        owner;                /* pid that opened the socket */

        struct mutex                sk_lock;        /* Protects .sk */
        struct sock __rcu        *sk;                /* Pointer to the session PPPoX socket */
        struct sock                *__sk;                /* Copy of .sk, for cleanup */
};

static int pppol2tp_xmit(struct ppp_channel *chan, struct sk_buff *skb);

static const struct ppp_channel_ops pppol2tp_chan_ops = {
        .start_xmit =  pppol2tp_xmit,
};

static const struct proto_ops pppol2tp_ops;

/* Retrieves the pppol2tp socket associated to a session. */
static struct sock *pppol2tp_session_get_sock(struct l2tp_session *session)
{
        struct pppol2tp_session *ps = l2tp_session_priv(session);

        return rcu_dereference(ps->sk);
}

/* Helpers to obtain tunnel/session contexts from sockets.
 */
static struct l2tp_session *pppol2tp_sock_to_session(struct sock *sk)
{
        struct l2tp_session *session;

        if (!sk)
                return NULL;

        rcu_read_lock();
        session = rcu_dereference_sk_user_data(sk);
        if (session && refcount_inc_not_zero(&session->ref_count)) {
                rcu_read_unlock();
                WARN_ON_ONCE(session->magic != L2TP_SESSION_MAGIC);
                return session;
        }
        rcu_read_unlock();

        return NULL;
}

/*****************************************************************************
 * Receive data handling
 *****************************************************************************/

/* Receive message. This is the recvmsg for the PPPoL2TP socket.
 */
static int pppol2tp_recvmsg(struct socket *sock, struct msghdr *msg,
                            size_t len, int flags)
{
        int err;
        struct sk_buff *skb;
        struct sock *sk = sock->sk;

        err = -EIO;
        if (sk->sk_state & PPPOX_BOUND)
                goto end;

        err = 0;
        skb = skb_recv_datagram(sk, flags, &err);
        if (!skb)
                goto end;

        if (len > skb->len)
                len = skb->len;
        else if (len < skb->len)
                msg->msg_flags |= MSG_TRUNC;

        err = skb_copy_datagram_msg(skb, 0, msg, len);
        if (likely(err == 0))
                err = len;

        kfree_skb(skb);
end:
        return err;
}

static void pppol2tp_recv(struct l2tp_session *session, struct sk_buff *skb, int data_len)
{
        struct sock *sk;

        /* If the socket is bound, send it in to PPP's input queue. Otherwise
         * queue it on the session socket.
         */
        rcu_read_lock();
        sk = pppol2tp_session_get_sock(session);
        if (!sk)
                goto no_sock;

        /* If the first two bytes are 0xFF03, consider that it is the PPP's
         * Address and Control fields and skip them. The L2TP module has always
         * worked this way, although, in theory, the use of these fields should
         * be negotiated and handled at the PPP layer. These fields are
         * constant: 0xFF is the All-Stations Address and 0x03 the Unnumbered
         * Information command with Poll/Final bit set to zero (RFC 1662).
         */
        if (pskb_may_pull(skb, 2) && skb->data[0] == PPP_ALLSTATIONS &&
            skb->data[1] == PPP_UI)
                skb_pull(skb, 2);

        if (sk->sk_state & PPPOX_BOUND) {
                struct pppox_sock *po;

                po = pppox_sk(sk);
                ppp_input(&po->chan, skb);
        } else {
                if (sock_queue_rcv_skb(sk, skb) < 0) {
                        atomic_long_inc(&session->stats.rx_errors);
                        kfree_skb(skb);
                }
        }
        rcu_read_unlock();

        return;

no_sock:
        rcu_read_unlock();
        pr_warn_ratelimited("%s: no socket in recv\n", session->name);
        kfree_skb(skb);
}

/************************************************************************
 * Transmit handling
 ***********************************************************************/

/* This is the sendmsg for the PPPoL2TP pppol2tp_session socket.  We come here
 * when a user application does a sendmsg() on the session socket. L2TP and
 * PPP headers must be inserted into the user's data.
 */
static int pppol2tp_sendmsg(struct socket *sock, struct msghdr *m,
                            size_t total_len)
{
        struct sock *sk = sock->sk;
        struct sk_buff *skb;
        int error;
        struct l2tp_session *session;
        struct l2tp_tunnel *tunnel;
        int uhlen;

        error = -ENOTCONN;
        if (sock_flag(sk, SOCK_DEAD) || !(sk->sk_state & PPPOX_CONNECTED))
                goto error;

        /* Get session and tunnel contexts */
        error = -EBADF;
        session = pppol2tp_sock_to_session(sk);
        if (!session)
                goto error;

        tunnel = session->tunnel;

        uhlen = (tunnel->encap == L2TP_ENCAPTYPE_UDP) ? sizeof(struct udphdr) : 0;

        /* Allocate a socket buffer */
        error = -ENOMEM;
        skb = sock_wmalloc(sk, NET_SKB_PAD + sizeof(struct iphdr) +
                           uhlen + session->hdr_len +
                           2 + total_len, /* 2 bytes for PPP_ALLSTATIONS & PPP_UI */
                           0, GFP_KERNEL);
        if (!skb)
                goto error_put_sess;

        /* Reserve space for headers. */
        skb_reserve(skb, NET_SKB_PAD);
        skb_reset_network_header(skb);
        skb_reserve(skb, sizeof(struct iphdr));
        skb_reset_transport_header(skb);
        skb_reserve(skb, uhlen);

        /* Add PPP header */
        skb->data[0] = PPP_ALLSTATIONS;
        skb->data[1] = PPP_UI;
        skb_put(skb, 2);

        /* Copy user data into skb */
        error = memcpy_from_msg(skb_put(skb, total_len), m, total_len);
        if (error < 0) {
                kfree_skb(skb);
                goto error_put_sess;
        }

        local_bh_disable();
        l2tp_xmit_skb(session, skb);
        local_bh_enable();

        l2tp_session_put(session);

        return total_len;

error_put_sess:
        l2tp_session_put(session);
error:
        return error;
}

/* Transmit function called by generic PPP driver.  Sends PPP frame
 * over PPPoL2TP socket.
 *
 * This is almost the same as pppol2tp_sendmsg(), but rather than
 * being called with a msghdr from userspace, it is called with a skb
 * from the kernel.
 *
 * The supplied skb from ppp doesn't have enough headroom for the
 * insertion of L2TP, UDP and IP headers so we need to allocate more
 * headroom in the skb. This will create a cloned skb. But we must be
 * careful in the error case because the caller will expect to free
 * the skb it supplied, not our cloned skb. So we take care to always
 * leave the original skb unfreed if we return an error.
 */
static int pppol2tp_xmit(struct ppp_channel *chan, struct sk_buff *skb)
{
        struct sock *sk = (struct sock *)chan->private;
        struct l2tp_session *session;
        struct l2tp_tunnel *tunnel;
        int uhlen, headroom;

        if (sock_flag(sk, SOCK_DEAD) || !(sk->sk_state & PPPOX_CONNECTED))
                goto abort;

        /* Get session and tunnel contexts from the socket */
        session = pppol2tp_sock_to_session(sk);
        if (!session)
                goto abort;

        tunnel = session->tunnel;

        uhlen = (tunnel->encap == L2TP_ENCAPTYPE_UDP) ? sizeof(struct udphdr) : 0;
        headroom = NET_SKB_PAD +
                   sizeof(struct iphdr) + /* IP header */
                   uhlen +                /* UDP header (if L2TP_ENCAPTYPE_UDP) */
                   session->hdr_len +        /* L2TP header */
                   2;                        /* 2 bytes for PPP_ALLSTATIONS & PPP_UI */
        if (skb_cow_head(skb, headroom))
                goto abort_put_sess;

        /* Setup PPP header */
        __skb_push(skb, 2);
        skb->data[0] = PPP_ALLSTATIONS;
        skb->data[1] = PPP_UI;

        local_bh_disable();
        l2tp_xmit_skb(session, skb);
        local_bh_enable();

        l2tp_session_put(session);

        return 1;

abort_put_sess:
        l2tp_session_put(session);
abort:
        /* Free the original skb */
        kfree_skb(skb);
        return 1;
}

/*****************************************************************************
 * Session (and tunnel control) socket create/destroy.
 *****************************************************************************/

/* Really kill the session socket. (Called from sock_put() if
 * refcnt == 0.)
 */
static void pppol2tp_session_destruct(struct sock *sk)
{
        skb_queue_purge(&sk->sk_receive_queue);
        skb_queue_purge(&sk->sk_write_queue);
}

static void pppol2tp_session_close(struct l2tp_session *session)
{
        struct pppol2tp_session *ps;

        ps = l2tp_session_priv(session);
        mutex_lock(&ps->sk_lock);
        ps->__sk = rcu_dereference_protected(ps->sk,
                                             lockdep_is_held(&ps->sk_lock));
        RCU_INIT_POINTER(ps->sk, NULL);
        mutex_unlock(&ps->sk_lock);
        if (ps->__sk) {
                /* detach socket */
                rcu_assign_sk_user_data(ps->__sk, NULL);
                sock_put(ps->__sk);

                /* drop ref taken when we referenced socket via sk_user_data */
                l2tp_session_put(session);
        }
}

/* Called when the PPPoX socket (session) is closed.
 */
static int pppol2tp_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct l2tp_session *session;
        int error;

        if (!sk)
                return 0;

        error = -EBADF;
        lock_sock(sk);
        if (sock_flag(sk, SOCK_DEAD) != 0)
                goto error;

        pppox_unbind_sock(sk);

        /* Signal the death of the socket. */
        sk->sk_state = PPPOX_DEAD;
        sock_orphan(sk);
        sock->sk = NULL;

        session = pppol2tp_sock_to_session(sk);
        if (session) {
                l2tp_session_delete(session);
                /* drop ref taken by pppol2tp_sock_to_session */
                l2tp_session_put(session);
        }

        release_sock(sk);

        sock_put(sk);

        return 0;

error:
        release_sock(sk);
        return error;
}

static struct proto pppol2tp_sk_proto = {
        .name          = "PPPOL2TP",
        .owner          = THIS_MODULE,
        .obj_size = sizeof(struct pppox_sock),
};

static int pppol2tp_backlog_recv(struct sock *sk, struct sk_buff *skb)
{
        int rc;

        rc = l2tp_udp_encap_recv(sk, skb);
        if (rc)
                kfree_skb(skb);

        return NET_RX_SUCCESS;
}

/* socket() handler. Initialize a new struct sock.
 */
static int pppol2tp_create(struct net *net, struct socket *sock, int kern)
{
        int error = -ENOMEM;
        struct sock *sk;

        sk = sk_alloc(net, PF_PPPOX, GFP_KERNEL, &pppol2tp_sk_proto, kern);
        if (!sk)
                goto out;

        sock_init_data(sock, sk);
        sock_set_flag(sk, SOCK_RCU_FREE);

        sock->state  = SS_UNCONNECTED;
        sock->ops    = &pppol2tp_ops;

        sk->sk_backlog_rcv = pppol2tp_backlog_recv;
        sk->sk_protocol           = PX_PROTO_OL2TP;
        sk->sk_family           = PF_PPPOX;
        sk->sk_state           = PPPOX_NONE;
        sk->sk_type           = SOCK_STREAM;
        sk->sk_destruct           = pppol2tp_session_destruct;

        error = 0;

out:
        return error;
}

static void pppol2tp_show(struct seq_file *m, void *arg)
{
        struct l2tp_session *session = arg;
        struct sock *sk;

        rcu_read_lock();
        sk = pppol2tp_session_get_sock(session);
        if (sk) {
                struct pppox_sock *po = pppox_sk(sk);

                seq_printf(m, "   interface %s\n", ppp_dev_name(&po->chan));
        }
        rcu_read_unlock();
}

static void pppol2tp_session_init(struct l2tp_session *session)
{
        struct pppol2tp_session *ps;

        session->recv_skb = pppol2tp_recv;
        session->session_close = pppol2tp_session_close;
        if (IS_ENABLED(CONFIG_L2TP_DEBUGFS))
                session->show = pppol2tp_show;

        ps = l2tp_session_priv(session);
        mutex_init(&ps->sk_lock);
        ps->owner = current->pid;
}

struct l2tp_connect_info {
        u8 version;
        int fd;
        u32 tunnel_id;
        u32 peer_tunnel_id;
        u32 session_id;
        u32 peer_session_id;
};

static int pppol2tp_sockaddr_get_info(const void *sa, int sa_len,
                                      struct l2tp_connect_info *info)
{
        switch (sa_len) {
        case sizeof(struct sockaddr_pppol2tp):
        {
                const struct sockaddr_pppol2tp *sa_v2in4 = sa;

                if (sa_v2in4->sa_protocol != PX_PROTO_OL2TP)
                        return -EINVAL;

                info->version = 2;
                info->fd = sa_v2in4->pppol2tp.fd;
                info->tunnel_id = sa_v2in4->pppol2tp.s_tunnel;
                info->peer_tunnel_id = sa_v2in4->pppol2tp.d_tunnel;
                info->session_id = sa_v2in4->pppol2tp.s_session;
                info->peer_session_id = sa_v2in4->pppol2tp.d_session;

                break;
        }
        case sizeof(struct sockaddr_pppol2tpv3):
        {
                const struct sockaddr_pppol2tpv3 *sa_v3in4 = sa;

                if (sa_v3in4->sa_protocol != PX_PROTO_OL2TP)
                        return -EINVAL;

                info->version = 3;
                info->fd = sa_v3in4->pppol2tp.fd;
                info->tunnel_id = sa_v3in4->pppol2tp.s_tunnel;
                info->peer_tunnel_id = sa_v3in4->pppol2tp.d_tunnel;
                info->session_id = sa_v3in4->pppol2tp.s_session;
                info->peer_session_id = sa_v3in4->pppol2tp.d_session;

                break;
        }
        case sizeof(struct sockaddr_pppol2tpin6):
        {
                const struct sockaddr_pppol2tpin6 *sa_v2in6 = sa;

                if (sa_v2in6->sa_protocol != PX_PROTO_OL2TP)
                        return -EINVAL;

                info->version = 2;
                info->fd = sa_v2in6->pppol2tp.fd;
                info->tunnel_id = sa_v2in6->pppol2tp.s_tunnel;
                info->peer_tunnel_id = sa_v2in6->pppol2tp.d_tunnel;
                info->session_id = sa_v2in6->pppol2tp.s_session;
                info->peer_session_id = sa_v2in6->pppol2tp.d_session;

                break;
        }
        case sizeof(struct sockaddr_pppol2tpv3in6):
        {
                const struct sockaddr_pppol2tpv3in6 *sa_v3in6 = sa;

                if (sa_v3in6->sa_protocol != PX_PROTO_OL2TP)
                        return -EINVAL;

                info->version = 3;
                info->fd = sa_v3in6->pppol2tp.fd;
                info->tunnel_id = sa_v3in6->pppol2tp.s_tunnel;
                info->peer_tunnel_id = sa_v3in6->pppol2tp.d_tunnel;
                info->session_id = sa_v3in6->pppol2tp.s_session;
                info->peer_session_id = sa_v3in6->pppol2tp.d_session;

                break;
        }
        default:
                return -EINVAL;
        }

        return 0;
}

/* Rough estimation of the maximum payload size a tunnel can transmit without
 * fragmenting at the lower IP layer. Assumes L2TPv2 with sequence
 * numbers and no IP option. Not quite accurate, but the result is mostly
 * unused anyway.
 */
static int pppol2tp_tunnel_mtu(const struct l2tp_tunnel *tunnel)
{
        int mtu;

        mtu = l2tp_tunnel_dst_mtu(tunnel);
        if (mtu <= PPPOL2TP_HEADER_OVERHEAD)
                return 1500 - PPPOL2TP_HEADER_OVERHEAD;

        return mtu - PPPOL2TP_HEADER_OVERHEAD;
}

static struct l2tp_tunnel *pppol2tp_tunnel_get(struct net *net,
                                               const struct l2tp_connect_info *info,
                                               bool *new_tunnel)
{
        struct l2tp_tunnel *tunnel;
        int error;

        *new_tunnel = false;

        tunnel = l2tp_tunnel_get(net, info->tunnel_id);

        /* Special case: create tunnel context if session_id and
         * peer_session_id is 0. Otherwise look up tunnel using supplied
         * tunnel id.
         */
        if (!info->session_id && !info->peer_session_id) {
                if (!tunnel) {
                        struct l2tp_tunnel_cfg tcfg = {
                                .encap = L2TP_ENCAPTYPE_UDP,
                        };

                        /* Prevent l2tp_tunnel_register() from trying to set up
                         * a kernel socket.
                         */
                        if (info->fd < 0)
                                return ERR_PTR(-EBADF);

                        error = l2tp_tunnel_create(info->fd,
                                                   info->version,
                                                   info->tunnel_id,
                                                   info->peer_tunnel_id, &tcfg,
                                                   &tunnel);
                        if (error < 0)
                                return ERR_PTR(error);

                        refcount_inc(&tunnel->ref_count);
                        error = l2tp_tunnel_register(tunnel, net, &tcfg);
                        if (error < 0) {
                                kfree(tunnel);
                                return ERR_PTR(error);
                        }

                        *new_tunnel = true;
                }
        } else {
                /* Error if we can't find the tunnel */
                if (!tunnel)
                        return ERR_PTR(-ENOENT);

                /* Error if socket is not prepped */
                if (!tunnel->sock) {
                        l2tp_tunnel_put(tunnel);
                        return ERR_PTR(-ENOENT);
                }
        }

        return tunnel;
}

/* connect() handler. Attach a PPPoX socket to a tunnel UDP socket
 */
static int pppol2tp_connect(struct socket *sock, struct sockaddr_unsized *uservaddr,
                            int sockaddr_len, int flags)
{
        struct sock *sk = sock->sk;
        struct pppox_sock *po = pppox_sk(sk);
        struct l2tp_session *session = NULL;
        struct l2tp_connect_info info;
        struct l2tp_tunnel *tunnel;
        struct pppol2tp_session *ps;
        struct l2tp_session_cfg cfg = { 0, };
        bool drop_refcnt = false;
        bool new_session = false;
        bool new_tunnel = false;
        int error;

        error = pppol2tp_sockaddr_get_info(uservaddr, sockaddr_len, &info);
        if (error < 0)
                return error;

        /* Don't bind if tunnel_id is 0 */
        if (!info.tunnel_id)
                return -EINVAL;

        tunnel = pppol2tp_tunnel_get(sock_net(sk), &info, &new_tunnel);
        if (IS_ERR(tunnel))
                return PTR_ERR(tunnel);

        lock_sock(sk);

        /* Check for already bound sockets */
        error = -EBUSY;
        if (sk->sk_state & PPPOX_CONNECTED)
                goto end;

        /* We don't supporting rebinding anyway */
        error = -EALREADY;
        if (sk->sk_user_data)
                goto end; /* socket is already attached */

        if (tunnel->peer_tunnel_id == 0)
                tunnel->peer_tunnel_id = info.peer_tunnel_id;

        session = l2tp_session_get(sock_net(sk), tunnel->sock, tunnel->version,
                                   info.tunnel_id, info.session_id);
        if (session) {
                drop_refcnt = true;

                if (session->pwtype != L2TP_PWTYPE_PPP) {
                        error = -EPROTOTYPE;
                        goto end;
                }

                ps = l2tp_session_priv(session);

                /* Using a pre-existing session is fine as long as it hasn't
                 * been connected yet.
                 */
                mutex_lock(&ps->sk_lock);
                if (rcu_dereference_protected(ps->sk,
                                              lockdep_is_held(&ps->sk_lock)) ||
                    ps->__sk) {
                        mutex_unlock(&ps->sk_lock);
                        error = -EEXIST;
                        goto end;
                }
        } else {
                cfg.pw_type = L2TP_PWTYPE_PPP;

                session = l2tp_session_create(sizeof(struct pppol2tp_session),
                                              tunnel, info.session_id,
                                              info.peer_session_id, &cfg);
                if (IS_ERR(session)) {
                        error = PTR_ERR(session);
                        goto end;
                }

                drop_refcnt = true;

                pppol2tp_session_init(session);
                ps = l2tp_session_priv(session);
                refcount_inc(&session->ref_count);

                mutex_lock(&ps->sk_lock);
                error = l2tp_session_register(session, tunnel);
                if (error < 0) {
                        mutex_unlock(&ps->sk_lock);
                        l2tp_session_put(session);
                        goto end;
                }

                new_session = true;
        }

        /* Special case: if source & dest session_id == 0x0000, this
         * socket is being created to manage the tunnel. Just set up
         * the internal context for use by ioctl() and sockopt()
         * handlers.
         */
        if (session->session_id == 0 && session->peer_session_id == 0) {
                error = 0;
                goto out_no_ppp;
        }

        /* Reserve enough headroom for the L2TP header with sequence numbers,
         * which is the largest possible. This is used by the PPP layer to set
         * the net device's hard_header_len at registration, which must be
         * sufficient regardless of whether sequence numbers are enabled later.
         */
        po->chan.hdrlen = PPPOL2TP_L2TP_HDR_SIZE_SEQ;

        po->chan.private = sk;
        po->chan.ops         = &pppol2tp_chan_ops;
        po->chan.mtu         = pppol2tp_tunnel_mtu(tunnel);
        po->chan.direct_xmit        = true;

        error = ppp_register_net_channel(sock_net(sk), &po->chan);
        if (error) {
                mutex_unlock(&ps->sk_lock);
                goto end;
        }

out_no_ppp:
        /* This is how we get the session context from the socket. */
        sock_hold(sk);
        rcu_assign_sk_user_data(sk, session);
        rcu_assign_pointer(ps->sk, sk);
        mutex_unlock(&ps->sk_lock);

        /* Keep the reference we've grabbed on the session: sk doesn't expect
         * the session to disappear. pppol2tp_session_close() is responsible
         * for dropping it.
         */
        drop_refcnt = false;

        sk->sk_state = PPPOX_CONNECTED;

end:
        if (error) {
                if (new_session)
                        l2tp_session_delete(session);
                if (new_tunnel)
                        l2tp_tunnel_delete(tunnel);
        }
        if (drop_refcnt)
                l2tp_session_put(session);
        l2tp_tunnel_put(tunnel);
        release_sock(sk);

        return error;
}

#ifdef CONFIG_L2TP_V3

/* Called when creating sessions via the netlink interface. */
static int pppol2tp_session_create(struct net *net, struct l2tp_tunnel *tunnel,
                                   u32 session_id, u32 peer_session_id,
                                   struct l2tp_session_cfg *cfg)
{
        int error;
        struct l2tp_session *session;

        /* Error if tunnel socket is not prepped */
        if (!tunnel->sock) {
                error = -ENOENT;
                goto err;
        }

        /* Allocate and initialize a new session context. */
        session = l2tp_session_create(sizeof(struct pppol2tp_session),
                                      tunnel, session_id,
                                      peer_session_id, cfg);
        if (IS_ERR(session)) {
                error = PTR_ERR(session);
                goto err;
        }

        pppol2tp_session_init(session);

        error = l2tp_session_register(session, tunnel);
        if (error < 0)
                goto err_sess;

        return 0;

err_sess:
        l2tp_session_put(session);
err:
        return error;
}

#endif /* CONFIG_L2TP_V3 */

/* getname() support.
 */
static int pppol2tp_getname(struct socket *sock, struct sockaddr *uaddr,
                            int peer)
{
        int len = 0;
        int error = 0;
        struct l2tp_session *session;
        struct l2tp_tunnel *tunnel;
        struct sock *sk = sock->sk;
        struct inet_sock *inet;
        struct pppol2tp_session *pls;

        error = -ENOTCONN;
        if (!sk)
                goto end;
        if (!(sk->sk_state & PPPOX_CONNECTED))
                goto end;

        error = -EBADF;
        session = pppol2tp_sock_to_session(sk);
        if (!session)
                goto end;

        pls = l2tp_session_priv(session);
        tunnel = session->tunnel;

        inet = inet_sk(tunnel->sock);
        if (tunnel->version == 2 && tunnel->sock->sk_family == AF_INET) {
                struct sockaddr_pppol2tp sp;

                len = sizeof(sp);
                memset(&sp, 0, len);
                sp.sa_family        = AF_PPPOX;
                sp.sa_protocol        = PX_PROTO_OL2TP;
                sp.pppol2tp.fd  = tunnel->fd;
                sp.pppol2tp.pid = pls->owner;
                sp.pppol2tp.s_tunnel = tunnel->tunnel_id;
                sp.pppol2tp.d_tunnel = tunnel->peer_tunnel_id;
                sp.pppol2tp.s_session = session->session_id;
                sp.pppol2tp.d_session = session->peer_session_id;
                sp.pppol2tp.addr.sin_family = AF_INET;
                sp.pppol2tp.addr.sin_port = inet->inet_dport;
                sp.pppol2tp.addr.sin_addr.s_addr = inet->inet_daddr;
                memcpy(uaddr, &sp, len);
#if IS_ENABLED(CONFIG_IPV6)
        } else if (tunnel->version == 2 && tunnel->sock->sk_family == AF_INET6) {
                struct sockaddr_pppol2tpin6 sp;

                len = sizeof(sp);
                memset(&sp, 0, len);
                sp.sa_family        = AF_PPPOX;
                sp.sa_protocol        = PX_PROTO_OL2TP;
                sp.pppol2tp.fd  = tunnel->fd;
                sp.pppol2tp.pid = pls->owner;
                sp.pppol2tp.s_tunnel = tunnel->tunnel_id;
                sp.pppol2tp.d_tunnel = tunnel->peer_tunnel_id;
                sp.pppol2tp.s_session = session->session_id;
                sp.pppol2tp.d_session = session->peer_session_id;
                sp.pppol2tp.addr.sin6_family = AF_INET6;
                sp.pppol2tp.addr.sin6_port = inet->inet_dport;
                memcpy(&sp.pppol2tp.addr.sin6_addr, &tunnel->sock->sk_v6_daddr,
                       sizeof(tunnel->sock->sk_v6_daddr));
                memcpy(uaddr, &sp, len);
        } else if (tunnel->version == 3 && tunnel->sock->sk_family == AF_INET6) {
                struct sockaddr_pppol2tpv3in6 sp;

                len = sizeof(sp);
                memset(&sp, 0, len);
                sp.sa_family        = AF_PPPOX;
                sp.sa_protocol        = PX_PROTO_OL2TP;
                sp.pppol2tp.fd  = tunnel->fd;
                sp.pppol2tp.pid = pls->owner;
                sp.pppol2tp.s_tunnel = tunnel->tunnel_id;
                sp.pppol2tp.d_tunnel = tunnel->peer_tunnel_id;
                sp.pppol2tp.s_session = session->session_id;
                sp.pppol2tp.d_session = session->peer_session_id;
                sp.pppol2tp.addr.sin6_family = AF_INET6;
                sp.pppol2tp.addr.sin6_port = inet->inet_dport;
                memcpy(&sp.pppol2tp.addr.sin6_addr, &tunnel->sock->sk_v6_daddr,
                       sizeof(tunnel->sock->sk_v6_daddr));
                memcpy(uaddr, &sp, len);
#endif
        } else if (tunnel->version == 3) {
                struct sockaddr_pppol2tpv3 sp;

                len = sizeof(sp);
                memset(&sp, 0, len);
                sp.sa_family        = AF_PPPOX;
                sp.sa_protocol        = PX_PROTO_OL2TP;
                sp.pppol2tp.fd  = tunnel->fd;
                sp.pppol2tp.pid = pls->owner;
                sp.pppol2tp.s_tunnel = tunnel->tunnel_id;
                sp.pppol2tp.d_tunnel = tunnel->peer_tunnel_id;
                sp.pppol2tp.s_session = session->session_id;
                sp.pppol2tp.d_session = session->peer_session_id;
                sp.pppol2tp.addr.sin_family = AF_INET;
                sp.pppol2tp.addr.sin_port = inet->inet_dport;
                sp.pppol2tp.addr.sin_addr.s_addr = inet->inet_daddr;
                memcpy(uaddr, &sp, len);
        }

        error = len;

        l2tp_session_put(session);
end:
        return error;
}

/****************************************************************************
 * ioctl() handlers.
 *
 * The PPPoX socket is created for L2TP sessions: tunnels have their own UDP
 * sockets. However, in order to control kernel tunnel features, we allow
 * userspace to create a special "tunnel" PPPoX socket which is used for
 * control only.  Tunnel PPPoX sockets have session_id == 0 and simply allow
 * the user application to issue L2TP setsockopt(), getsockopt() and ioctl()
 * calls.
 ****************************************************************************/

static void pppol2tp_copy_stats(struct pppol2tp_ioc_stats *dest,
                                const struct l2tp_stats *stats)
{
        memset(dest, 0, sizeof(*dest));

        dest->tx_packets = atomic_long_read(&stats->tx_packets);
        dest->tx_bytes = atomic_long_read(&stats->tx_bytes);
        dest->tx_errors = atomic_long_read(&stats->tx_errors);
        dest->rx_packets = atomic_long_read(&stats->rx_packets);
        dest->rx_bytes = atomic_long_read(&stats->rx_bytes);
        dest->rx_seq_discards = atomic_long_read(&stats->rx_seq_discards);
        dest->rx_oos_packets = atomic_long_read(&stats->rx_oos_packets);
        dest->rx_errors = atomic_long_read(&stats->rx_errors);
}

static int pppol2tp_tunnel_copy_stats(struct pppol2tp_ioc_stats *stats,
                                      struct l2tp_tunnel *tunnel)
{
        struct l2tp_session *session;

        if (!stats->session_id) {
                pppol2tp_copy_stats(stats, &tunnel->stats);
                return 0;
        }

        /* If session_id is set, search the corresponding session in the
         * context of this tunnel and record the session's statistics.
         */
        session = l2tp_session_get(tunnel->l2tp_net, tunnel->sock, tunnel->version,
                                   tunnel->tunnel_id, stats->session_id);
        if (!session)
                return -EBADR;

        if (session->pwtype != L2TP_PWTYPE_PPP) {
                l2tp_session_put(session);
                return -EBADR;
        }

        pppol2tp_copy_stats(stats, &session->stats);
        l2tp_session_put(session);

        return 0;
}

static int pppol2tp_ioctl(struct socket *sock, unsigned int cmd,
                          unsigned long arg)
{
        struct pppol2tp_ioc_stats stats;
        struct l2tp_session *session;
        int err = 0;

        session = pppol2tp_sock_to_session(sock->sk);

        /* Validate session presence and magic integrity ONLY for commands
         * that belong to L2TP and require a valid session.
         */
        switch (cmd) {
        case PPPIOCGMRU:
        case PPPIOCGFLAGS:
        case PPPIOCSMRU:
        case PPPIOCSFLAGS:
        case PPPIOCGL2TPSTATS:
                if (!session)
                        return -ENOTCONN;

                if (session->magic != L2TP_SESSION_MAGIC) {
                        l2tp_session_put(session);
                        return -EBADF;
                }
                break;
        default:
                break;
        }

        switch (cmd) {
        case PPPIOCGMRU:
        case PPPIOCGFLAGS:
                /* Not defined for tunnels */
                if (!session->session_id && !session->peer_session_id) {
                        err = -ENOSYS;
                        break;
                }

                if (put_user(0, (int __user *)arg)) {
                        err = -EFAULT;
                        break;
                }
                break;

        case PPPIOCSMRU:
        case PPPIOCSFLAGS:
                /* Not defined for tunnels */
                if (!session->session_id && !session->peer_session_id) {
                        err = -ENOSYS;
                        break;
                }

                if (!access_ok((int __user *)arg, sizeof(int))) {
                        err = -EFAULT;
                        break;
                }
                break;

        case PPPIOCGL2TPSTATS:
                /* Session 0 represents the parent tunnel */
                if (!session->session_id && !session->peer_session_id) {
                        u32 session_id;

                        if (copy_from_user(&stats, (void __user *)arg,
                                           sizeof(stats))) {
                                err = -EFAULT;
                                break;
                        }

                        session_id = stats.session_id;
                        err = pppol2tp_tunnel_copy_stats(&stats,
                                                         session->tunnel);
                        if (err < 0)
                                break;

                        stats.session_id = session_id;
                } else {
                        pppol2tp_copy_stats(&stats, &session->stats);
                        stats.session_id = session->session_id;
                }
                stats.tunnel_id = session->tunnel->tunnel_id;
                stats.using_ipsec = l2tp_tunnel_uses_xfrm(session->tunnel);

                if (copy_to_user((void __user *)arg, &stats, sizeof(stats))) {
                        err = -EFAULT;
                        break;
                }
                break;

        default:
                err = -ENOIOCTLCMD;
                break;
        }

        if (session)
                l2tp_session_put(session);

        return err;
}

/*****************************************************************************
 * setsockopt() / getsockopt() support.
 *
 * The PPPoX socket is created for L2TP sessions: tunnels have their own UDP
 * sockets. In order to control kernel tunnel features, we allow userspace to
 * create a special "tunnel" PPPoX socket which is used for control only.
 * Tunnel PPPoX sockets have session_id == 0 and simply allow the user
 * application to issue L2TP setsockopt(), getsockopt() and ioctl() calls.
 *****************************************************************************/

/* Tunnel setsockopt() helper.
 */
static int pppol2tp_tunnel_setsockopt(struct sock *sk,
                                      struct l2tp_tunnel *tunnel,
                                      int optname, int val)
{
        int err = 0;

        switch (optname) {
        case PPPOL2TP_SO_DEBUG:
                /* Tunnel debug flags option is deprecated */
                break;

        default:
                err = -ENOPROTOOPT;
                break;
        }

        return err;
}

/* Session setsockopt helper.
 */
static int pppol2tp_session_setsockopt(struct sock *sk,
                                       struct l2tp_session *session,
                                       int optname, int val)
{
        int err = 0;

        switch (optname) {
        case PPPOL2TP_SO_RECVSEQ:
                if (val != 0 && val != 1) {
                        err = -EINVAL;
                        break;
                }
                session->recv_seq = !!val;
                break;

        case PPPOL2TP_SO_SENDSEQ:
                if (val != 0 && val != 1) {
                        err = -EINVAL;
                        break;
                }
                session->send_seq = !!val;
                l2tp_session_set_header_len(session, session->tunnel->version,
                                            session->tunnel->encap);
                break;

        case PPPOL2TP_SO_LNSMODE:
                if (val != 0 && val != 1) {
                        err = -EINVAL;
                        break;
                }
                session->lns_mode = !!val;
                break;

        case PPPOL2TP_SO_DEBUG:
                /* Session debug flags option is deprecated */
                break;

        case PPPOL2TP_SO_REORDERTO:
                session->reorder_timeout = msecs_to_jiffies(val);
                break;

        default:
                err = -ENOPROTOOPT;
                break;
        }

        return err;
}

/* Main setsockopt() entry point.
 * Does API checks, then calls either the tunnel or session setsockopt
 * handler, according to whether the PPPoL2TP socket is a for a regular
 * session or the special tunnel type.
 */
static int pppol2tp_setsockopt(struct socket *sock, int level, int optname,
                               sockptr_t optval, unsigned int optlen)
{
        struct sock *sk = sock->sk;
        struct l2tp_session *session;
        struct l2tp_tunnel *tunnel;
        int val;
        int err;

        if (level != SOL_PPPOL2TP)
                return -EINVAL;

        if (optlen < sizeof(int))
                return -EINVAL;

        if (copy_from_sockptr(&val, optval, sizeof(int)))
                return -EFAULT;

        err = -ENOTCONN;
        if (!sk->sk_user_data)
                goto end;

        /* Get session context from the socket */
        err = -EBADF;
        session = pppol2tp_sock_to_session(sk);
        if (!session)
                goto end;

        /* Special case: if session_id == 0x0000, treat as operation on tunnel
         */
        if (session->session_id == 0 && session->peer_session_id == 0) {
                tunnel = session->tunnel;
                err = pppol2tp_tunnel_setsockopt(sk, tunnel, optname, val);
        } else {
                err = pppol2tp_session_setsockopt(sk, session, optname, val);
        }

        l2tp_session_put(session);
end:
        return err;
}

/* Tunnel getsockopt helper. Called with sock locked.
 */
static int pppol2tp_tunnel_getsockopt(struct sock *sk,
                                      struct l2tp_tunnel *tunnel,
                                      int optname, int *val)
{
        int err = 0;

        switch (optname) {
        case PPPOL2TP_SO_DEBUG:
                /* Tunnel debug flags option is deprecated */
                *val = 0;
                break;

        default:
                err = -ENOPROTOOPT;
                break;
        }

        return err;
}

/* Session getsockopt helper. Called with sock locked.
 */
static int pppol2tp_session_getsockopt(struct sock *sk,
                                       struct l2tp_session *session,
                                       int optname, int *val)
{
        int err = 0;

        switch (optname) {
        case PPPOL2TP_SO_RECVSEQ:
                *val = session->recv_seq;
                break;

        case PPPOL2TP_SO_SENDSEQ:
                *val = session->send_seq;
                break;

        case PPPOL2TP_SO_LNSMODE:
                *val = session->lns_mode;
                break;

        case PPPOL2TP_SO_DEBUG:
                /* Session debug flags option is deprecated */
                *val = 0;
                break;

        case PPPOL2TP_SO_REORDERTO:
                *val = (int)jiffies_to_msecs(session->reorder_timeout);
                break;

        default:
                err = -ENOPROTOOPT;
        }

        return err;
}

/* Main getsockopt() entry point.
 * Does API checks, then calls either the tunnel or session getsockopt
 * handler, according to whether the PPPoX socket is a for a regular session
 * or the special tunnel type.
 */
static int pppol2tp_getsockopt(struct socket *sock, int level, int optname,
                               char __user *optval, int __user *optlen)
{
        struct sock *sk = sock->sk;
        struct l2tp_session *session;
        struct l2tp_tunnel *tunnel;
        int val, len;
        int err;

        if (level != SOL_PPPOL2TP)
                return -EINVAL;

        if (get_user(len, optlen))
                return -EFAULT;

        if (len < 0)
                return -EINVAL;

        len = min_t(unsigned int, len, sizeof(int));

        err = -ENOTCONN;
        if (!sk->sk_user_data)
                goto end;

        /* Get the session context */
        err = -EBADF;
        session = pppol2tp_sock_to_session(sk);
        if (!session)
                goto end;

        /* Special case: if session_id == 0x0000, treat as operation on tunnel */
        if (session->session_id == 0 && session->peer_session_id == 0) {
                tunnel = session->tunnel;
                err = pppol2tp_tunnel_getsockopt(sk, tunnel, optname, &val);
                if (err)
                        goto end_put_sess;
        } else {
                err = pppol2tp_session_getsockopt(sk, session, optname, &val);
                if (err)
                        goto end_put_sess;
        }

        err = -EFAULT;
        if (put_user(len, optlen))
                goto end_put_sess;

        if (copy_to_user((void __user *)optval, &val, len))
                goto end_put_sess;

        err = 0;

end_put_sess:
        l2tp_session_put(session);
end:
        return err;
}

/*****************************************************************************
 * /proc filesystem for debug
 * Since the original pppol2tp driver provided /proc/net/pppol2tp for
 * L2TPv2, we dump only L2TPv2 tunnels and sessions here.
 *****************************************************************************/

#ifdef CONFIG_PROC_FS

struct pppol2tp_seq_data {
        struct seq_net_private p;
        unsigned long tkey;                /* lookup key of current tunnel */
        unsigned long skey;                /* lookup key of current session */
        struct l2tp_tunnel *tunnel;
        struct l2tp_session *session;        /* NULL means get next tunnel */
};

static void pppol2tp_next_tunnel(struct net *net, struct pppol2tp_seq_data *pd)
{
        /* Drop reference taken during previous invocation */
        if (pd->tunnel)
                l2tp_tunnel_put(pd->tunnel);

        for (;;) {
                pd->tunnel = l2tp_tunnel_get_next(net, &pd->tkey);
                pd->tkey++;

                /* Only accept L2TPv2 tunnels */
                if (!pd->tunnel || pd->tunnel->version == 2)
                        return;

                l2tp_tunnel_put(pd->tunnel);
        }
}

static void pppol2tp_next_session(struct net *net, struct pppol2tp_seq_data *pd)
{
        /* Drop reference taken during previous invocation */
        if (pd->session)
                l2tp_session_put(pd->session);

        pd->session = l2tp_session_get_next(net, pd->tunnel->sock,
                                            pd->tunnel->version,
                                            pd->tunnel->tunnel_id, &pd->skey);
        pd->skey++;

        if (!pd->session) {
                pd->skey = 0;
                pppol2tp_next_tunnel(net, pd);
        }
}

static void *pppol2tp_seq_start(struct seq_file *m, loff_t *offs)
{
        struct pppol2tp_seq_data *pd = SEQ_START_TOKEN;
        loff_t pos = *offs;
        struct net *net;

        if (!pos)
                goto out;

        if (WARN_ON(!m->private)) {
                pd = NULL;
                goto out;
        }

        pd = m->private;
        net = seq_file_net(m);

        if (!pd->tunnel)
                pppol2tp_next_tunnel(net, pd);
        else
                pppol2tp_next_session(net, pd);

        /* NULL tunnel and session indicates end of list */
        if (!pd->tunnel && !pd->session)
                pd = NULL;

out:
        return pd;
}

static void *pppol2tp_seq_next(struct seq_file *m, void *v, loff_t *pos)
{
        (*pos)++;
        return NULL;
}

static void pppol2tp_seq_stop(struct seq_file *p, void *v)
{
        struct pppol2tp_seq_data *pd = v;

        if (!pd || pd == SEQ_START_TOKEN)
                return;

        /* Drop reference taken by last invocation of pppol2tp_next_session()
         * or pppol2tp_next_tunnel().
         */
        if (pd->session) {
                l2tp_session_put(pd->session);
                pd->session = NULL;
        }
        if (pd->tunnel) {
                l2tp_tunnel_put(pd->tunnel);
                pd->tunnel = NULL;
        }
}

static void pppol2tp_seq_tunnel_show(struct seq_file *m, void *v)
{
        struct l2tp_tunnel *tunnel = v;

        seq_printf(m, "\nTUNNEL '%s', %c %d\n",
                   tunnel->name,
                   tunnel->sock ? 'Y' : 'N',
                   refcount_read(&tunnel->ref_count) - 1);
        seq_printf(m, " %08x %ld/%ld/%ld %ld/%ld/%ld\n",
                   0,
                   atomic_long_read(&tunnel->stats.tx_packets),
                   atomic_long_read(&tunnel->stats.tx_bytes),
                   atomic_long_read(&tunnel->stats.tx_errors),
                   atomic_long_read(&tunnel->stats.rx_packets),
                   atomic_long_read(&tunnel->stats.rx_bytes),
                   atomic_long_read(&tunnel->stats.rx_errors));
}

static void pppol2tp_seq_session_show(struct seq_file *m, void *v)
{
        struct l2tp_session *session = v;
        struct l2tp_tunnel *tunnel = session->tunnel;
        unsigned char state;
        char user_data_ok;
        struct sock *sk;
        u32 ip = 0;
        u16 port = 0;

        if (tunnel->sock) {
                struct inet_sock *inet = inet_sk(tunnel->sock);

                ip = ntohl(inet->inet_saddr);
                port = ntohs(inet->inet_sport);
        }

        rcu_read_lock();
        sk = pppol2tp_session_get_sock(session);
        if (sk) {
                state = sk->sk_state;
                user_data_ok = (session == sk->sk_user_data) ? 'Y' : 'N';
        } else {
                state = 0;
                user_data_ok = 'N';
        }

        seq_printf(m, "  SESSION '%s' %08X/%d %04X/%04X -> %04X/%04X %d %c\n",
                   session->name, ip, port,
                   tunnel->tunnel_id,
                   session->session_id,
                   tunnel->peer_tunnel_id,
                   session->peer_session_id,
                   state, user_data_ok);
        seq_printf(m, "   0/0/%c/%c/%s %08x %u\n",
                   session->recv_seq ? 'R' : '-',
                   session->send_seq ? 'S' : '-',
                   session->lns_mode ? "LNS" : "LAC",
                   0,
                   jiffies_to_msecs(session->reorder_timeout));
        seq_printf(m, "   %u/%u %ld/%ld/%ld %ld/%ld/%ld\n",
                   session->nr, session->ns,
                   atomic_long_read(&session->stats.tx_packets),
                   atomic_long_read(&session->stats.tx_bytes),
                   atomic_long_read(&session->stats.tx_errors),
                   atomic_long_read(&session->stats.rx_packets),
                   atomic_long_read(&session->stats.rx_bytes),
                   atomic_long_read(&session->stats.rx_errors));

        if (sk) {
                struct pppox_sock *po = pppox_sk(sk);

                seq_printf(m, "   interface %s\n", ppp_dev_name(&po->chan));
        }
        rcu_read_unlock();
}

static int pppol2tp_seq_show(struct seq_file *m, void *v)
{
        struct pppol2tp_seq_data *pd = v;

        /* display header on line 1 */
        if (v == SEQ_START_TOKEN) {
                seq_puts(m, "PPPoL2TP driver info, " PPPOL2TP_DRV_VERSION "\n");
                seq_puts(m, "TUNNEL name, user-data-ok session-count\n");
                seq_puts(m, " debug tx-pkts/bytes/errs rx-pkts/bytes/errs\n");
                seq_puts(m, "  SESSION name, addr/port src-tid/sid dest-tid/sid state user-data-ok\n");
                seq_puts(m, "   mtu/mru/rcvseq/sendseq/lns debug reorderto\n");
                seq_puts(m, "   nr/ns tx-pkts/bytes/errs rx-pkts/bytes/errs\n");
                goto out;
        }

        if (!pd->session)
                pppol2tp_seq_tunnel_show(m, pd->tunnel);
        else
                pppol2tp_seq_session_show(m, pd->session);

out:
        return 0;
}

static const struct seq_operations pppol2tp_seq_ops = {
        .start                = pppol2tp_seq_start,
        .next                = pppol2tp_seq_next,
        .stop                = pppol2tp_seq_stop,
        .show                = pppol2tp_seq_show,
};
#endif /* CONFIG_PROC_FS */

/*****************************************************************************
 * Network namespace
 *****************************************************************************/

static __net_init int pppol2tp_init_net(struct net *net)
{
        struct proc_dir_entry *pde;
        int err = 0;

        pde = proc_create_net("pppol2tp", 0444, net->proc_net,
                              &pppol2tp_seq_ops, sizeof(struct pppol2tp_seq_data));
        if (!pde) {
                err = -ENOMEM;
                goto out;
        }

out:
        return err;
}

static __net_exit void pppol2tp_exit_net(struct net *net)
{
        remove_proc_entry("pppol2tp", net->proc_net);
}

static struct pernet_operations pppol2tp_net_ops = {
        .init = pppol2tp_init_net,
        .exit = pppol2tp_exit_net,
};

/*****************************************************************************
 * Init and cleanup
 *****************************************************************************/

static const struct proto_ops pppol2tp_ops = {
        .family                = AF_PPPOX,
        .owner                = THIS_MODULE,
        .release        = pppol2tp_release,
        .bind                = sock_no_bind,
        .connect        = pppol2tp_connect,
        .socketpair        = sock_no_socketpair,
        .accept                = sock_no_accept,
        .getname        = pppol2tp_getname,
        .poll                = datagram_poll,
        .listen                = sock_no_listen,
        .shutdown        = sock_no_shutdown,
        .setsockopt        = pppol2tp_setsockopt,
        .getsockopt        = pppol2tp_getsockopt,
        .sendmsg        = pppol2tp_sendmsg,
        .recvmsg        = pppol2tp_recvmsg,
        .mmap                = sock_no_mmap,
        .ioctl                = pppox_ioctl,
#ifdef CONFIG_COMPAT
        .compat_ioctl = pppox_compat_ioctl,
#endif
};

static const struct pppox_proto pppol2tp_proto = {
        .create                = pppol2tp_create,
        .ioctl                = pppol2tp_ioctl,
        .owner                = THIS_MODULE,
};

#ifdef CONFIG_L2TP_V3

static const struct l2tp_nl_cmd_ops pppol2tp_nl_cmd_ops = {
        .session_create        = pppol2tp_session_create,
        .session_delete        = l2tp_session_delete,
};

#endif /* CONFIG_L2TP_V3 */

static int __init pppol2tp_init(void)
{
        int err;

        err = register_pernet_device(&pppol2tp_net_ops);
        if (err)
                goto out;

        err = proto_register(&pppol2tp_sk_proto, 0);
        if (err)
                goto out_unregister_pppol2tp_pernet;

        err = register_pppox_proto(PX_PROTO_OL2TP, &pppol2tp_proto);
        if (err)
                goto out_unregister_pppol2tp_proto;

#ifdef CONFIG_L2TP_V3
        err = l2tp_nl_register_ops(L2TP_PWTYPE_PPP, &pppol2tp_nl_cmd_ops);
        if (err)
                goto out_unregister_pppox;
#endif

        pr_info("PPPoL2TP kernel driver, %s\n", PPPOL2TP_DRV_VERSION);

out:
        return err;

#ifdef CONFIG_L2TP_V3
out_unregister_pppox:
        unregister_pppox_proto(PX_PROTO_OL2TP);
#endif
out_unregister_pppol2tp_proto:
        proto_unregister(&pppol2tp_sk_proto);
out_unregister_pppol2tp_pernet:
        unregister_pernet_device(&pppol2tp_net_ops);
        goto out;
}

static void __exit pppol2tp_exit(void)
{
#ifdef CONFIG_L2TP_V3
        l2tp_nl_unregister_ops(L2TP_PWTYPE_PPP);
#endif
        unregister_pppox_proto(PX_PROTO_OL2TP);
        proto_unregister(&pppol2tp_sk_proto);
        unregister_pernet_device(&pppol2tp_net_ops);
}

module_init(pppol2tp_init);
module_exit(pppol2tp_exit);

MODULE_AUTHOR("James Chapman <jchapman@katalix.com>");
MODULE_DESCRIPTION("PPP over L2TP over UDP");
MODULE_LICENSE("GPL");
MODULE_VERSION(PPPOL2TP_DRV_VERSION);
MODULE_ALIAS_NET_PF_PROTO(PF_PPPOX, PX_PROTO_OL2TP);
MODULE_ALIAS_L2TP_PWTYPE(7);




























































    7 








    7 




    7 





















    2 













    2 



    2 

    2 



    2 

    2 


















































   33 



    2 


   31 











   16 






    4 





    3 

















    7 















   10 






   10 







    4 
    4 













































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/module.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_socket.h>
#include <net/inet_sock.h>
#include <net/tcp.h>

struct nft_socket {
        enum nft_socket_keys                key:8;
        u8                                level;                /* cgroupv2 level to extract */
        u8                                level_user;        /* cgroupv2 level provided by userspace */
        u8                                len;
        union {
                u8                        dreg;
        };
};

static void nft_socket_wildcard(const struct nft_pktinfo *pkt,
                                struct nft_regs *regs, struct sock *sk,
                                u32 *dest)
{
        switch (nft_pf(pkt)) {
        case NFPROTO_IPV4:
                nft_reg_store8(dest, inet_sk(sk)->inet_rcv_saddr == 0);
                break;
#if IS_ENABLED(CONFIG_NF_TABLES_IPV6)
        case NFPROTO_IPV6:
                nft_reg_store8(dest, ipv6_addr_any(&sk->sk_v6_rcv_saddr));
                break;
#endif
        default:
                regs->verdict.code = NFT_BREAK;
                return;
        }
}

#ifdef CONFIG_SOCK_CGROUP_DATA
static noinline bool
nft_sock_get_eval_cgroupv2(u32 *dest, struct sock *sk, const struct nft_pktinfo *pkt, u32 level)
{
        struct cgroup *cgrp;
        u64 cgid;

        if (!sk_fullsock(sk))
                return false;

        cgrp = cgroup_ancestor(sock_cgroup_ptr(&sk->sk_cgrp_data), level);
        if (!cgrp)
                return false;

        cgid = cgroup_id(cgrp);
        memcpy(dest, &cgid, sizeof(u64));
        return true;
}

/* process context only, uses current->nsproxy. */
static noinline int nft_socket_cgroup_subtree_level(void)
{
        struct cgroup *cgrp = cgroup_get_from_path("/");
        int level;

        if (IS_ERR(cgrp))
                return PTR_ERR(cgrp);

        level = cgrp->level;

        cgroup_put(cgrp);

        if (level > 255)
                return -ERANGE;

        if (WARN_ON_ONCE(level < 0))
                return -EINVAL;

        return level;
}
#endif

static struct sock *nft_socket_do_lookup(const struct nft_pktinfo *pkt)
{
        const struct net_device *indev = nft_in(pkt);
        const struct sk_buff *skb = pkt->skb;
        struct sock *sk = NULL;

        if (!indev)
                return NULL;

        switch (nft_pf(pkt)) {
        case NFPROTO_IPV4:
                sk = nf_sk_lookup_slow_v4(nft_net(pkt), skb, indev);
                break;
#if IS_ENABLED(CONFIG_NF_TABLES_IPV6)
        case NFPROTO_IPV6:
                sk = nf_sk_lookup_slow_v6(nft_net(pkt), skb, indev);
                break;
#endif
        default:
                WARN_ON_ONCE(1);
                break;
        }

        return sk;
}

static void nft_socket_eval(const struct nft_expr *expr,
                            struct nft_regs *regs,
                            const struct nft_pktinfo *pkt)
{
        const struct nft_socket *priv = nft_expr_priv(expr);
        struct sk_buff *skb = pkt->skb;
        struct sock *sk = skb->sk;
        u32 *dest = &regs->data[priv->dreg];

        if (sk && !net_eq(nft_net(pkt), sock_net(sk)))
                sk = NULL;

        if (!sk)
                sk = nft_socket_do_lookup(pkt);

        if (!sk) {
                regs->verdict.code = NFT_BREAK;
                return;
        }

        switch(priv->key) {
        case NFT_SOCKET_TRANSPARENT:
                nft_reg_store8(dest, inet_sk_transparent(sk));
                break;
        case NFT_SOCKET_MARK:
                if (sk_fullsock(sk)) {
                        *dest = READ_ONCE(sk->sk_mark);
                } else {
                        regs->verdict.code = NFT_BREAK;
                        goto out_put_sk;
                }
                break;
        case NFT_SOCKET_WILDCARD:
                if (!sk_fullsock(sk)) {
                        regs->verdict.code = NFT_BREAK;
                        goto out_put_sk;
                }
                nft_socket_wildcard(pkt, regs, sk, dest);
                break;
#ifdef CONFIG_SOCK_CGROUP_DATA
        case NFT_SOCKET_CGROUPV2:
                if (!nft_sock_get_eval_cgroupv2(dest, sk, pkt, priv->level)) {
                        regs->verdict.code = NFT_BREAK;
                        goto out_put_sk;
                }
                break;
#endif
        default:
                WARN_ON(1);
                regs->verdict.code = NFT_BREAK;
        }

out_put_sk:
        if (sk != skb->sk)
                sock_gen_put(sk);
}

static const struct nla_policy nft_socket_policy[NFTA_SOCKET_MAX + 1] = {
        [NFTA_SOCKET_KEY]                = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_SOCKET_DREG]                = NLA_POLICY_MAX(NLA_BE32, NFT_REG32_MAX),
        [NFTA_SOCKET_LEVEL]                = NLA_POLICY_MAX(NLA_BE32, 255),
};

static int nft_socket_init(const struct nft_ctx *ctx,
                           const struct nft_expr *expr,
                           const struct nlattr * const tb[])
{
        struct nft_socket *priv = nft_expr_priv(expr);
        unsigned int len;

        if (!tb[NFTA_SOCKET_DREG] || !tb[NFTA_SOCKET_KEY])
                return -EINVAL;

        switch(ctx->family) {
        case NFPROTO_IPV4:
#if IS_ENABLED(CONFIG_NF_TABLES_IPV6)
        case NFPROTO_IPV6:
#endif
        case NFPROTO_INET:
                break;
        default:
                return -EOPNOTSUPP;
        }

        priv->key = ntohl(nla_get_be32(tb[NFTA_SOCKET_KEY]));
        switch(priv->key) {
        case NFT_SOCKET_TRANSPARENT:
        case NFT_SOCKET_WILDCARD:
                len = sizeof(u8);
                break;
        case NFT_SOCKET_MARK:
                len = sizeof(u32);
                break;
#ifdef CONFIG_SOCK_CGROUP_DATA
        case NFT_SOCKET_CGROUPV2: {
                unsigned int level;
                int err;

                if (!tb[NFTA_SOCKET_LEVEL])
                        return -EINVAL;

                level = ntohl(nla_get_be32(tb[NFTA_SOCKET_LEVEL]));
                if (level > 255)
                        return -EOPNOTSUPP;

                err = nft_socket_cgroup_subtree_level();
                if (err < 0)
                        return err;

                priv->level_user = level;

                level += err;
                /* Implies a giant cgroup tree */
                if (level > 255)
                        return -EOPNOTSUPP;

                priv->level = level;
                len = sizeof(u64);
                break;
        }
#endif
        default:
                return -EOPNOTSUPP;
        }

        priv->len = len;
        return nft_parse_register_store(ctx, tb[NFTA_SOCKET_DREG], &priv->dreg,
                                        NULL, NFT_DATA_VALUE, len);
}

static int nft_socket_dump(struct sk_buff *skb,
                           const struct nft_expr *expr, bool reset)
{
        const struct nft_socket *priv = nft_expr_priv(expr);

        if (nla_put_be32(skb, NFTA_SOCKET_KEY, htonl(priv->key)))
                return -1;
        if (nft_dump_register(skb, NFTA_SOCKET_DREG, priv->dreg))
                return -1;
        if (priv->key == NFT_SOCKET_CGROUPV2 &&
            nla_put_be32(skb, NFTA_SOCKET_LEVEL, htonl(priv->level_user)))
                return -1;
        return 0;
}

static int nft_socket_validate(const struct nft_ctx *ctx,
                               const struct nft_expr *expr)
{
        if (ctx->family != NFPROTO_IPV4 &&
            ctx->family != NFPROTO_IPV6 &&
            ctx->family != NFPROTO_INET)
                return -EOPNOTSUPP;

        return nft_chain_validate_hooks(ctx->chain,
                                        (1 << NF_INET_PRE_ROUTING) |
                                        (1 << NF_INET_LOCAL_IN) |
                                        (1 << NF_INET_LOCAL_OUT));
}

static struct nft_expr_type nft_socket_type;
static const struct nft_expr_ops nft_socket_ops = {
        .type                = &nft_socket_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_socket)),
        .eval                = nft_socket_eval,
        .init                = nft_socket_init,
        .dump                = nft_socket_dump,
        .validate        = nft_socket_validate,
};

static struct nft_expr_type nft_socket_type __read_mostly = {
        .name                = "socket",
        .ops                = &nft_socket_ops,
        .policy                = nft_socket_policy,
        .maxattr        = NFTA_SOCKET_MAX,
        .owner                = THIS_MODULE,
};

static int __init nft_socket_module_init(void)
{
        return nft_register_expr(&nft_socket_type);
}

static void __exit nft_socket_module_exit(void)
{
        nft_unregister_expr(&nft_socket_type);
}

module_init(nft_socket_module_init);
module_exit(nft_socket_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Máté Eckl");
MODULE_DESCRIPTION("nf_tables socket match module");
MODULE_ALIAS_NFT_EXPR("socket");

























    9 


    4 



    8 




















   59 








   72 










   72 




   72 



   72 


   72 




   74 





















    2 






    2 


    2 























    8 





    8 

    8 
    8 








































   10 











    8 

    2 









   10 











   35 

























   46 









   13 

   35 



   13 
   47 









   72 








   12 







   72 






















   76 








    2 





   74 

   75 



   74 






   75 




   74 
    2 









   75 










   71 
    3 






    2 








   13 
   72 








   72 










   55 



   18 







   14 









   45 
   26 
   13 
   35 




   47 


   69 



   60 
   17 







   11 




   13 









    1 





   10 






















    9 


   65 
    7 






   42 


    2 






    4 


   89 





    1 


    2 


    7 




    1 


    6 





    5 





    7 




    2 


    6 




    2 



    3 

   12 

   10 








   14 

    3 

























    5 











   74 

























  141 

















  141 



   77 

    2 










   78 






   74 


    5 


    1 


















    5 









   77 




















    2 












   76 







   10 
   75 
































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
// SPDX-License-Identifier: GPL-2.0-or-later
/* AF_RXRPC sendmsg() implementation.
 *
 * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/net.h>
#include <linux/gfp.h>
#include <linux/skbuff.h>
#include <linux/export.h>
#include <linux/sched/signal.h>

#include <net/sock.h>
#include <net/af_rxrpc.h>
#include "ar-internal.h"

/*
 * Propose an abort to be made in the I/O thread.
 */
bool rxrpc_propose_abort(struct rxrpc_call *call, s32 abort_code, int error,
                         enum rxrpc_abort_reason why)
{
        _enter("{%d},%d,%d,%u", call->debug_id, abort_code, error, why);

        if (!call->send_abort && !rxrpc_call_is_complete(call)) {
                call->send_abort_why = why;
                call->send_abort_err = error;
                call->send_abort_seq = 0;
                trace_rxrpc_abort_call(call, abort_code);
                /* Request abort locklessly vs rxrpc_input_call_event(). */
                smp_store_release(&call->send_abort, abort_code);
                rxrpc_poke_call(call, rxrpc_call_poke_abort);
                return true;
        }

        return false;
}

/*
 * Wait for a call to become connected.  Interruption here doesn't cause the
 * call to be aborted.
 */
static int rxrpc_wait_to_be_connected(struct rxrpc_call *call, long *timeo)
{
        DECLARE_WAITQUEUE(myself, current);
        int ret = 0;

        _enter("%d", call->debug_id);

        if (rxrpc_call_state(call) != RXRPC_CALL_CLIENT_AWAIT_CONN)
                goto no_wait;

        add_wait_queue_exclusive(&call->waitq, &myself);

        for (;;) {
                switch (call->interruptibility) {
                case RXRPC_INTERRUPTIBLE:
                case RXRPC_PREINTERRUPTIBLE:
                        set_current_state(TASK_INTERRUPTIBLE);
                        break;
                case RXRPC_UNINTERRUPTIBLE:
                default:
                        set_current_state(TASK_UNINTERRUPTIBLE);
                        break;
                }

                if (rxrpc_call_state(call) != RXRPC_CALL_CLIENT_AWAIT_CONN)
                        break;
                if ((call->interruptibility == RXRPC_INTERRUPTIBLE ||
                     call->interruptibility == RXRPC_PREINTERRUPTIBLE) &&
                    signal_pending(current)) {
                        ret = sock_intr_errno(*timeo);
                        break;
                }
                *timeo = schedule_timeout(*timeo);
        }

        remove_wait_queue(&call->waitq, &myself);
        __set_current_state(TASK_RUNNING);

no_wait:
        if (ret == 0 && rxrpc_call_is_complete(call))
                ret = call->error;

        _leave(" = %d", ret);
        return ret;
}

/*
 * Return true if there's sufficient Tx queue space.
 */
static bool rxrpc_check_tx_space(struct rxrpc_call *call, rxrpc_seq_t *_tx_win)
{
        rxrpc_seq_t tx_bottom = READ_ONCE(call->tx_bottom);

        if (_tx_win)
                *_tx_win = tx_bottom;
        return call->send_top - tx_bottom < 256;
}

/*
 * Wait for space to appear in the Tx queue or a signal to occur.
 */
static int rxrpc_wait_for_tx_window_intr(struct rxrpc_sock *rx,
                                         struct rxrpc_call *call,
                                         long *timeo)
{
        for (;;) {
                set_current_state(TASK_INTERRUPTIBLE);
                if (rxrpc_check_tx_space(call, NULL))
                        return 0;

                if (rxrpc_call_is_complete(call))
                        return call->error;

                if (signal_pending(current))
                        return sock_intr_errno(*timeo);

                trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);
                *timeo = schedule_timeout(*timeo);
        }
}

/*
 * Wait for space to appear in the Tx queue uninterruptibly, but with
 * a timeout of 2*RTT if no progress was made and a signal occurred.
 */
static int rxrpc_wait_for_tx_window_waitall(struct rxrpc_sock *rx,
                                            struct rxrpc_call *call)
{
        rxrpc_seq_t tx_start, tx_win;
        signed long rtt, timeout;

        rtt = READ_ONCE(call->srtt_us) >> 3;
        rtt = usecs_to_jiffies(rtt) * 2;
        if (rtt < 2)
                rtt = 2;

        timeout = rtt;
        tx_start = READ_ONCE(call->tx_bottom);

        for (;;) {
                set_current_state(TASK_UNINTERRUPTIBLE);

                if (rxrpc_check_tx_space(call, &tx_win))
                        return 0;

                if (rxrpc_call_is_complete(call))
                        return call->error;

                if (timeout == 0 &&
                    tx_win == tx_start && signal_pending(current))
                        return -EINTR;

                if (tx_win != tx_start) {
                        timeout = rtt;
                        tx_start = tx_win;
                }

                trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);
                timeout = schedule_timeout(timeout);
        }
}

/*
 * Wait for space to appear in the Tx queue uninterruptibly.
 */
static int rxrpc_wait_for_tx_window_nonintr(struct rxrpc_sock *rx,
                                            struct rxrpc_call *call,
                                            long *timeo)
{
        for (;;) {
                set_current_state(TASK_UNINTERRUPTIBLE);
                if (rxrpc_check_tx_space(call, NULL))
                        return 0;

                if (rxrpc_call_is_complete(call))
                        return call->error;

                trace_rxrpc_txqueue(call, rxrpc_txqueue_wait);
                *timeo = schedule_timeout(*timeo);
        }
}

/*
 * wait for space to appear in the transmit/ACK window
 * - caller holds the socket locked
 */
static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
                                    struct rxrpc_call *call,
                                    long *timeo,
                                    bool waitall)
{
        DECLARE_WAITQUEUE(myself, current);
        int ret;

        _enter(",{%u,%u,%u}",
               call->tx_bottom, call->tx_top, call->tx_winsize);

        add_wait_queue(&call->waitq, &myself);

        switch (call->interruptibility) {
        case RXRPC_INTERRUPTIBLE:
                if (waitall)
                        ret = rxrpc_wait_for_tx_window_waitall(rx, call);
                else
                        ret = rxrpc_wait_for_tx_window_intr(rx, call, timeo);
                break;
        case RXRPC_PREINTERRUPTIBLE:
        case RXRPC_UNINTERRUPTIBLE:
        default:
                ret = rxrpc_wait_for_tx_window_nonintr(rx, call, timeo);
                break;
        }

        remove_wait_queue(&call->waitq, &myself);
        set_current_state(TASK_RUNNING);
        _leave(" = %d", ret);
        return ret;
}

/*
 * Notify the owner of the call that the transmit phase is ended and the last
 * packet has been queued.
 */
static void rxrpc_notify_end_tx(struct rxrpc_sock *rx, struct rxrpc_call *call,
                                rxrpc_notify_end_tx_t notify_end_tx)
{
        if (notify_end_tx)
                notify_end_tx(&rx->sk, call, call->user_call_ID);
}

/*
 * Queue a DATA packet for transmission, set the resend timeout and send
 * the packet immediately.  Returns the error from rxrpc_send_data_packet()
 * in case the caller wants to do something with it.
 */
static void rxrpc_queue_packet(struct rxrpc_sock *rx, struct rxrpc_call *call,
                               struct rxrpc_txbuf *txb,
                               rxrpc_notify_end_tx_t notify_end_tx)
{
        struct rxrpc_txqueue *sq = call->send_queue;
        rxrpc_seq_t seq = txb->seq;
        bool poke, last = txb->flags & RXRPC_LAST_PACKET;
        int ix = seq & RXRPC_TXQ_MASK;
        rxrpc_inc_stat(call->rxnet, stat_tx_data);

        ASSERTCMP(txb->seq, ==, call->send_top + 1);

        if (last)
                trace_rxrpc_txqueue(call, rxrpc_txqueue_queue_last);
        else
                trace_rxrpc_txqueue(call, rxrpc_txqueue_queue);

        if (WARN_ON_ONCE(sq->bufs[ix]))
                trace_rxrpc_tq(call, sq, seq, rxrpc_tq_queue_dup);
        else
                trace_rxrpc_tq(call, sq, seq, rxrpc_tq_queue);

        /* Add the packet to the call's output buffer */
        poke = (READ_ONCE(call->tx_bottom) == call->send_top);
        sq->bufs[ix] = txb;
        /* Order send_top after the queue->next pointer and txb content. */
        smp_store_release(&call->send_top, seq);
        if (last) {
                set_bit(RXRPC_CALL_TX_NO_MORE, &call->flags);
                rxrpc_notify_end_tx(rx, call, notify_end_tx);
                call->send_queue = NULL;
        }

        if (poke)
                rxrpc_poke_call(call, rxrpc_call_poke_start);
}

/*
 * Allocate a new txqueue unit and add it to the transmission queue.
 */
static int rxrpc_alloc_txqueue(struct sock *sk, struct rxrpc_call *call)
{
        struct rxrpc_txqueue *tq;

        tq = kzalloc_obj(*tq, sk->sk_allocation);
        if (!tq)
                return -ENOMEM;

        tq->xmit_ts_base = KTIME_MIN;
        for (int i = 0; i < RXRPC_NR_TXQUEUE; i++)
                tq->segment_xmit_ts[i] = UINT_MAX;

        if (call->send_queue) {
                tq->qbase = call->send_top + 1;
                call->send_queue->next = tq;
                call->send_queue = tq;
        } else if (WARN_ON(call->tx_queue)) {
                kfree(tq);
                return -ENOMEM;
        } else {
                /* We start at seq 1, so pretend seq 0 is hard-acked. */
                tq->nr_reported_acks = 1;
                tq->segment_acked = 1UL;
                tq->qbase = 0;
                call->tx_qbase = 0;
                call->send_queue = tq;
                call->tx_qtail = tq;
                call->tx_queue = tq;
        }

        trace_rxrpc_tq(call, tq, call->send_top, rxrpc_tq_alloc);
        return 0;
}

/*
 * send data through a socket
 * - must be called in process context
 * - The caller holds the call user access mutex, but not the socket lock.
 */
static int rxrpc_send_data(struct rxrpc_sock *rx,
                           struct rxrpc_call *call,
                           struct msghdr *msg, size_t len,
                           rxrpc_notify_end_tx_t notify_end_tx,
                           bool *_dropped_lock)
{
        struct rxrpc_txbuf *txb;
        struct sock *sk = &rx->sk;
        enum rxrpc_call_state state;
        long timeo;
        bool more = msg->msg_flags & MSG_MORE;
        int ret, copied = 0;

        if (test_bit(RXRPC_CALL_TX_NO_MORE, &call->flags)) {
                trace_rxrpc_abort(call->debug_id, rxrpc_sendmsg_late_send,
                                  call->cid, call->call_id, call->rx_consumed,
                                  0, -EPROTO);
                return -EPROTO;
        }

        timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);

        ret = rxrpc_wait_to_be_connected(call, &timeo);
        if (ret < 0)
                return ret;

        if (call->conn->state == RXRPC_CONN_CLIENT_UNSECURED) {
                ret = rxrpc_init_client_conn_security(call->conn);
                if (ret < 0)
                        return ret;
        }

        /* this should be in poll */
        sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);

reload:
        txb = call->tx_pending;
        call->tx_pending = NULL;
        if (txb)
                rxrpc_see_txbuf(txb, rxrpc_txbuf_see_send_more);

        ret = -EPIPE;
        if (sk->sk_shutdown & SEND_SHUTDOWN)
                goto maybe_error;
        state = rxrpc_call_state(call);
        ret = -ESHUTDOWN;
        if (state >= RXRPC_CALL_COMPLETE)
                goto maybe_error;
        ret = -EPROTO;
        if (state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
            state != RXRPC_CALL_SERVER_ACK_REQUEST &&
            state != RXRPC_CALL_SERVER_SEND_REPLY) {
                /* Request phase complete for this client call */
                trace_rxrpc_abort(call->debug_id, rxrpc_sendmsg_late_send,
                                  call->cid, call->call_id, call->rx_consumed,
                                  0, -EPROTO);
                goto maybe_error;
        }

        ret = -EMSGSIZE;
        if (call->tx_total_len != -1) {
                if (len - copied > call->tx_total_len)
                        goto maybe_error;
                if (!more && len - copied != call->tx_total_len)
                        goto maybe_error;
        }

        do {
                if (!txb) {
                        size_t remain;

                        _debug("alloc");

                        if (!rxrpc_check_tx_space(call, NULL))
                                goto wait_for_space;

                        /* See if we need to begin/extend the Tx queue. */
                        if (!call->send_queue || !((call->send_top + 1) & RXRPC_TXQ_MASK)) {
                                ret = rxrpc_alloc_txqueue(sk, call);
                                if (ret < 0)
                                        goto maybe_error;
                        }

                        /* Work out the maximum size of a packet.  Assume that
                         * the security header is going to be in the padded
                         * region (enc blocksize), but the trailer is not.
                         */
                        remain = more ? INT_MAX : msg_data_left(msg);
                        txb = call->conn->security->alloc_txbuf(call, remain, sk->sk_allocation);
                        if (!txb) {
                                ret = -ENOMEM;
                                goto maybe_error;
                        }
                }

                _debug("append");

                /* append next segment of data to the current buffer */
                if (msg_data_left(msg) > 0) {
                        size_t copy = umin(txb->space, msg_data_left(msg));

                        _debug("add %zu", copy);
                        if (!copy_from_iter_full(txb->data + txb->offset,
                                                 copy, &msg->msg_iter))
                                goto efault;
                        _debug("added");
                        txb->space -= copy;
                        txb->len += copy;
                        txb->offset += copy;
                        copied += copy;
                        if (call->tx_total_len != -1)
                                call->tx_total_len -= copy;
                }

                /* check for the far side aborting the call or a network error
                 * occurring */
                if (rxrpc_call_is_complete(call))
                        goto call_terminated;

                /* add the packet to the send queue if it's now full */
                if (!txb->space ||
                    (msg_data_left(msg) == 0 && !more)) {
                        if (msg_data_left(msg) == 0 && !more)
                                txb->flags |= RXRPC_LAST_PACKET;

                        ret = call->security->secure_packet(call, txb);
                        if (ret < 0)
                                goto out;
                        rxrpc_queue_packet(rx, call, txb, notify_end_tx);
                        txb = NULL;
                }
        } while (msg_data_left(msg) > 0);

success:
        ret = copied;
        if (rxrpc_call_is_complete(call) &&
            call->error < 0)
                ret = call->error;
out:
        call->tx_pending = txb;
        _leave(" = %d", ret);
        return ret;

call_terminated:
        rxrpc_put_txbuf(txb, rxrpc_txbuf_put_send_aborted);
        _leave(" = %d", call->error);
        return call->error;

maybe_error:
        if (copied)
                goto success;
        goto out;

efault:
        ret = -EFAULT;
        goto out;

wait_for_space:
        ret = -EAGAIN;
        if (msg->msg_flags & MSG_DONTWAIT)
                goto maybe_error;
        mutex_unlock(&call->user_mutex);
        *_dropped_lock = true;
        ret = rxrpc_wait_for_tx_window(rx, call, &timeo,
                                       msg->msg_flags & MSG_WAITALL);
        if (ret < 0)
                goto maybe_error;
        if (call->interruptibility == RXRPC_INTERRUPTIBLE) {
                if (mutex_lock_interruptible(&call->user_mutex) < 0) {
                        ret = sock_intr_errno(timeo);
                        goto maybe_error;
                }
        } else {
                mutex_lock(&call->user_mutex);
        }
        *_dropped_lock = false;
        goto reload;
}

/*
 * extract control messages from the sendmsg() control buffer
 */
static int rxrpc_sendmsg_cmsg(struct msghdr *msg, struct rxrpc_send_params *p)
{
        struct cmsghdr *cmsg;
        bool got_user_ID = false;
        int len;

        if (msg->msg_controllen == 0)
                return -EINVAL;

        for_each_cmsghdr(cmsg, msg) {
                if (!CMSG_OK(msg, cmsg))
                        return -EINVAL;

                len = cmsg->cmsg_len - sizeof(struct cmsghdr);
                _debug("CMSG %d, %d, %d",
                       cmsg->cmsg_level, cmsg->cmsg_type, len);

                if (cmsg->cmsg_level != SOL_RXRPC)
                        continue;

                switch (cmsg->cmsg_type) {
                case RXRPC_USER_CALL_ID:
                        if (msg->msg_flags & MSG_CMSG_COMPAT) {
                                if (len != sizeof(u32))
                                        return -EINVAL;
                                p->call.user_call_ID = *(u32 *)CMSG_DATA(cmsg);
                        } else {
                                if (len != sizeof(unsigned long))
                                        return -EINVAL;
                                p->call.user_call_ID = *(unsigned long *)
                                        CMSG_DATA(cmsg);
                        }
                        got_user_ID = true;
                        break;

                case RXRPC_ABORT:
                        if (p->command != RXRPC_CMD_SEND_DATA)
                                return -EINVAL;
                        p->command = RXRPC_CMD_SEND_ABORT;
                        if (len != sizeof(p->abort_code))
                                return -EINVAL;
                        p->abort_code = *(unsigned int *)CMSG_DATA(cmsg);
                        if (p->abort_code == 0)
                                return -EINVAL;
                        break;

                case RXRPC_CHARGE_ACCEPT:
                        if (p->command != RXRPC_CMD_SEND_DATA)
                                return -EINVAL;
                        p->command = RXRPC_CMD_CHARGE_ACCEPT;
                        if (len != 0)
                                return -EINVAL;
                        break;

                case RXRPC_EXCLUSIVE_CALL:
                        p->exclusive = true;
                        if (len != 0)
                                return -EINVAL;
                        break;

                case RXRPC_UPGRADE_SERVICE:
                        p->upgrade = true;
                        if (len != 0)
                                return -EINVAL;
                        break;

                case RXRPC_TX_LENGTH:
                        if (p->call.tx_total_len != -1 || len != sizeof(__s64))
                                return -EINVAL;
                        p->call.tx_total_len = *(__s64 *)CMSG_DATA(cmsg);
                        if (p->call.tx_total_len < 0)
                                return -EINVAL;
                        break;

                case RXRPC_SET_CALL_TIMEOUT:
                        if (len & 3 || len < 4 || len > 12)
                                return -EINVAL;
                        memcpy(&p->call.timeouts, CMSG_DATA(cmsg), len);
                        p->call.nr_timeouts = len / 4;
                        if (p->call.timeouts.hard > INT_MAX / HZ)
                                return -ERANGE;
                        if (p->call.nr_timeouts >= 2 && p->call.timeouts.idle > 60 * 60 * 1000)
                                return -ERANGE;
                        if (p->call.nr_timeouts >= 3 && p->call.timeouts.normal > 60 * 60 * 1000)
                                return -ERANGE;
                        break;

                default:
                        return -EINVAL;
                }
        }

        if (!got_user_ID)
                return -EINVAL;
        if (p->call.tx_total_len != -1 && p->command != RXRPC_CMD_SEND_DATA)
                return -EINVAL;
        _leave(" = 0");
        return 0;
}

/*
 * Create a new client call for sendmsg().
 * - Called with the socket lock held, which it must release.
 * - If it returns a call, the call's lock will need releasing by the caller.
 */
static struct rxrpc_call *
rxrpc_new_client_call_for_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg,
                                  struct rxrpc_send_params *p)
        __releases(&rx->sk.sk_lock)
        __acquires(&call->user_mutex)
{
        struct rxrpc_conn_parameters cp;
        struct rxrpc_peer *peer;
        struct rxrpc_call *call;
        struct key *key;

        DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx, msg->msg_name);

        _enter("");

        if (!msg->msg_name) {
                release_sock(&rx->sk);
                return ERR_PTR(-EDESTADDRREQ);
        }

        peer = rxrpc_lookup_peer(rx->local, srx, GFP_KERNEL);
        if (!peer) {
                release_sock(&rx->sk);
                return ERR_PTR(-ENOMEM);
        }

        key = rx->key;
        if (key && !rx->key->payload.data[0])
                key = NULL;

        memset(&cp, 0, sizeof(cp));
        cp.local                = rx->local;
        cp.peer                        = peer;
        cp.key                        = key;
        cp.security_level        = rx->min_sec_level;
        cp.exclusive                = rx->exclusive | p->exclusive;
        cp.upgrade                = p->upgrade;
        cp.service_id                = srx->srx_service;
        call = rxrpc_new_client_call(rx, &cp, &p->call, GFP_KERNEL,
                                     atomic_inc_return(&rxrpc_debug_id));
        /* The socket is now unlocked */

        rxrpc_put_peer(peer, rxrpc_peer_put_application);
        _leave(" = %p\n", call);
        return call;
}

/*
 * send a message forming part of a client call through an RxRPC socket
 * - caller holds the socket locked
 * - the socket may be either a client socket or a server socket
 */
int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len)
{
        struct rxrpc_call *call;
        bool dropped_lock = false;
        int ret;

        struct rxrpc_send_params p = {
                .call.tx_total_len        = -1,
                .call.user_call_ID        = 0,
                .call.nr_timeouts        = 0,
                .call.interruptibility        = RXRPC_INTERRUPTIBLE,
                .abort_code                = 0,
                .command                = RXRPC_CMD_SEND_DATA,
                .exclusive                = false,
                .upgrade                = false,
        };

        _enter("");

        ret = rxrpc_sendmsg_cmsg(msg, &p);
        if (ret < 0)
                goto error_release_sock;

        if (p.command == RXRPC_CMD_CHARGE_ACCEPT) {
                ret = -EINVAL;
                if (rx->sk.sk_state != RXRPC_SERVER_LISTENING)
                        goto error_release_sock;
                ret = rxrpc_user_charge_accept(rx, p.call.user_call_ID);
                goto error_release_sock;
        }

        call = rxrpc_find_call_by_user_ID(rx, p.call.user_call_ID);
        if (!call) {
                ret = -EBADSLT;
                if (p.command != RXRPC_CMD_SEND_DATA)
                        goto error_release_sock;
                call = rxrpc_new_client_call_for_sendmsg(rx, msg, &p);
                /* The socket is now unlocked... */
                if (IS_ERR(call))
                        return PTR_ERR(call);
                /* ... and we have the call lock. */
                p.call.nr_timeouts = 0;
                ret = 0;
                if (rxrpc_call_is_complete(call))
                        goto out_put_unlock;
        } else {
                switch (rxrpc_call_state(call)) {
                case RXRPC_CALL_CLIENT_AWAIT_CONN:
                case RXRPC_CALL_SERVER_RECV_REQUEST:
                        if (p.command == RXRPC_CMD_SEND_ABORT)
                                break;
                        fallthrough;
                case RXRPC_CALL_UNINITIALISED:
                case RXRPC_CALL_SERVER_PREALLOC:
                        rxrpc_put_call(call, rxrpc_call_put_sendmsg);
                        ret = -EBUSY;
                        goto error_release_sock;
                default:
                        break;
                }

                ret = mutex_lock_interruptible(&call->user_mutex);
                release_sock(&rx->sk);
                if (ret < 0) {
                        ret = -ERESTARTSYS;
                        goto error_put;
                }

                if (p.call.tx_total_len != -1) {
                        ret = -EINVAL;
                        if (call->tx_total_len != -1 ||
                            call->tx_pending ||
                            call->tx_top != 0)
                                goto out_put_unlock;
                        call->tx_total_len = p.call.tx_total_len;
                }
        }

        switch (p.call.nr_timeouts) {
        case 3:
                WRITE_ONCE(call->next_rx_timo, p.call.timeouts.normal);
                fallthrough;
        case 2:
                WRITE_ONCE(call->next_req_timo, p.call.timeouts.idle);
                fallthrough;
        case 1:
                if (p.call.timeouts.hard > 0) {
                        ktime_t delay = ms_to_ktime(p.call.timeouts.hard * MSEC_PER_SEC);

                        WRITE_ONCE(call->expect_term_by,
                                   ktime_add(p.call.timeouts.hard,
                                             ktime_get_real()));
                        trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_hard);
                        rxrpc_poke_call(call, rxrpc_call_poke_set_timeout);

                }
                break;
        }

        if (rxrpc_call_is_complete(call)) {
                /* it's too late for this call */
                ret = -ESHUTDOWN;
                goto out_put_unlock;
        }

        switch (p.command) {
        case RXRPC_CMD_SEND_ABORT:
                rxrpc_propose_abort(call, p.abort_code, -ECONNABORTED,
                                    rxrpc_abort_call_sendmsg);
                ret = 0;
                break;
        case RXRPC_CMD_SEND_DATA:
                ret = rxrpc_send_data(rx, call, msg, len, NULL, &dropped_lock);
                break;
        default:
                ret = -EINVAL;
                break;
        }

out_put_unlock:
        if (!dropped_lock)
                mutex_unlock(&call->user_mutex);
error_put:
        rxrpc_put_call(call, rxrpc_call_put_sendmsg);
        _leave(" = %d", ret);
        return ret;

error_release_sock:
        release_sock(&rx->sk);
        return ret;
}

/**
 * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
 * @sock: The socket the call is on
 * @call: The call to send data through
 * @msg: The data to send
 * @len: The amount of data to send
 * @notify_end_tx: Notification that the last packet is queued.
 *
 * Allow a kernel service to send data on a call.  The call must be in an state
 * appropriate to sending data.  No control data should be supplied in @msg,
 * nor should an address be supplied.  MSG_MORE should be flagged if there's
 * more data to come, otherwise this data will end the transmission phase.
 *
 * Return: %0 if successful and a negative error code otherwise.
 */
int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call,
                           struct msghdr *msg, size_t len,
                           rxrpc_notify_end_tx_t notify_end_tx)
{
        bool dropped_lock = false;
        int ret;

        _enter("{%d},", call->debug_id);

        ASSERTCMP(msg->msg_name, ==, NULL);
        ASSERTCMP(msg->msg_control, ==, NULL);

        mutex_lock(&call->user_mutex);

        ret = rxrpc_send_data(rxrpc_sk(sock->sk), call, msg, len,
                              notify_end_tx, &dropped_lock);
        if (ret == -ESHUTDOWN)
                ret = call->error;

        if (!dropped_lock)
                mutex_unlock(&call->user_mutex);
        _leave(" = %d", ret);
        return ret;
}
EXPORT_SYMBOL(rxrpc_kernel_send_data);

/**
 * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
 * @sock: The socket the call is on
 * @call: The call to be aborted
 * @abort_code: The abort code to stick into the ABORT packet
 * @error: Local error value
 * @why: Indication as to why.
 *
 * Allow a kernel service to abort a call if it's still in an abortable state.
 *
 * Return: %true if the call was aborted, %false if it was already complete.
 */
bool rxrpc_kernel_abort_call(struct socket *sock, struct rxrpc_call *call,
                             u32 abort_code, int error, enum rxrpc_abort_reason why)
{
        bool aborted;

        _enter("{%d},%d,%d,%u", call->debug_id, abort_code, error, why);

        mutex_lock(&call->user_mutex);
        aborted = rxrpc_propose_abort(call, abort_code, error, why);
        mutex_unlock(&call->user_mutex);
        return aborted;
}
EXPORT_SYMBOL(rxrpc_kernel_abort_call);

/**
 * rxrpc_kernel_set_tx_length - Set the total Tx length on a call
 * @sock: The socket the call is on
 * @call: The call to be informed
 * @tx_total_len: The amount of data to be transmitted for this call
 *
 * Allow a kernel service to set the total transmit length on a call.  This
 * allows buffer-to-packet encrypt-and-copy to be performed.
 *
 * This function is primarily for use for setting the reply length since the
 * request length can be set when beginning the call.
 */
void rxrpc_kernel_set_tx_length(struct socket *sock, struct rxrpc_call *call,
                                s64 tx_total_len)
{
        WARN_ON(call->tx_total_len != -1);
        call->tx_total_len = tx_total_len;
}
EXPORT_SYMBOL(rxrpc_kernel_set_tx_length);






















   10 











   10 













































    9 





    2 























































































   11 

















    1 



   11 




    1 










   11 
   11 
   11 






   13 




    1 







    3 










   10 







    2 























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
 */

#include <rdma/rdma_netlink.h>
#include <net/addrconf.h>
#include "rxe.h"
#include "rxe_loc.h"
#include "rxe_net.h"
#include "rxe_ns.h"

MODULE_AUTHOR("Bob Pearson, Frank Zago, John Groves, Kamal Heib");
MODULE_DESCRIPTION("Soft RDMA transport");
MODULE_LICENSE("Dual BSD/GPL");

/* free resources for a rxe device all objects created for this device must
 * have been destroyed
 */
void rxe_dealloc(struct ib_device *ib_dev)
{
        struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev);

        rxe_pool_cleanup(&rxe->uc_pool);
        rxe_pool_cleanup(&rxe->pd_pool);
        rxe_pool_cleanup(&rxe->ah_pool);
        rxe_pool_cleanup(&rxe->srq_pool);
        rxe_pool_cleanup(&rxe->qp_pool);
        rxe_pool_cleanup(&rxe->cq_pool);
        rxe_pool_cleanup(&rxe->mr_pool);
        rxe_pool_cleanup(&rxe->mw_pool);

        WARN_ON(!RB_EMPTY_ROOT(&rxe->mcg_tree));

        mutex_destroy(&rxe->usdev_lock);
}

static const struct ib_device_ops rxe_ib_dev_odp_ops = {
        .advise_mr = rxe_ib_advise_mr,
};

/* initialize rxe device parameters */
static void rxe_init_device_param(struct rxe_dev *rxe, struct net_device *ndev)
{
        rxe->max_inline_data                        = RXE_MAX_INLINE_DATA;

        rxe->attr.vendor_id                        = RXE_VENDOR_ID;
        rxe->attr.max_mr_size                        = RXE_MAX_MR_SIZE;
        rxe->attr.page_size_cap                        = RXE_PAGE_SIZE_CAP;
        rxe->attr.max_qp                        = RXE_MAX_QP;
        rxe->attr.max_qp_wr                        = RXE_MAX_QP_WR;
        rxe->attr.device_cap_flags                = RXE_DEVICE_CAP_FLAGS;
        rxe->attr.kernel_cap_flags                = IBK_ALLOW_USER_UNREG;
        rxe->attr.max_send_sge                        = RXE_MAX_SGE;
        rxe->attr.max_recv_sge                        = RXE_MAX_SGE;
        rxe->attr.max_sge_rd                        = RXE_MAX_SGE_RD;
        rxe->attr.max_cq                        = RXE_MAX_CQ;
        rxe->attr.max_cqe                        = (1 << RXE_MAX_LOG_CQE) - 1;
        rxe->attr.max_mr                        = RXE_MAX_MR;
        rxe->attr.max_mw                        = RXE_MAX_MW;
        rxe->attr.max_pd                        = RXE_MAX_PD;
        rxe->attr.max_qp_rd_atom                = RXE_MAX_QP_RD_ATOM;
        rxe->attr.max_res_rd_atom                = RXE_MAX_RES_RD_ATOM;
        rxe->attr.max_qp_init_rd_atom                = RXE_MAX_QP_INIT_RD_ATOM;
        rxe->attr.atomic_cap                        = IB_ATOMIC_HCA;
        rxe->attr.max_mcast_grp                        = RXE_MAX_MCAST_GRP;
        rxe->attr.max_mcast_qp_attach                = RXE_MAX_MCAST_QP_ATTACH;
        rxe->attr.max_total_mcast_qp_attach        = RXE_MAX_TOT_MCAST_QP_ATTACH;
        rxe->attr.max_ah                        = RXE_MAX_AH;
        rxe->attr.max_srq                        = RXE_MAX_SRQ;
        rxe->attr.max_srq_wr                        = RXE_MAX_SRQ_WR;
        rxe->attr.max_srq_sge                        = RXE_MAX_SRQ_SGE;
        rxe->attr.max_fast_reg_page_list_len        = RXE_MAX_FMR_PAGE_LIST_LEN;
        rxe->attr.max_pkeys                        = RXE_MAX_PKEYS;
        rxe->attr.local_ca_ack_delay                = RXE_LOCAL_CA_ACK_DELAY;

        if (ndev->addr_len) {
                memcpy(rxe->raw_gid, ndev->dev_addr,
                        min_t(unsigned int, ndev->addr_len, ETH_ALEN));
        } else {
                /*
                 * This device does not have a HW address, but
                 * connection mangagement requires a unique gid.
                 */
                eth_random_addr(rxe->raw_gid);
        }

        addrconf_addr_eui48((unsigned char *)&rxe->attr.sys_image_guid,
                        rxe->raw_gid);

        rxe->max_ucontext                        = RXE_MAX_UCONTEXT;

        if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
                rxe->attr.kernel_cap_flags |= IBK_ON_DEMAND_PAGING;

                /* IB_ODP_SUPPORT_IMPLICIT is not supported right now. */
                rxe->attr.odp_caps.general_caps |= IB_ODP_SUPPORT;

                rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND;
                rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_RECV;
                rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;

                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND;
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV;
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE;
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ;
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC;
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_FLUSH;
                rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC_WRITE;

                /* set handler for ODP prefetching API - ibv_advise_mr(3) */
                ib_set_device_ops(&rxe->ib_dev, &rxe_ib_dev_odp_ops);
        }
}

/* initialize port attributes */
static void rxe_init_port_param(struct rxe_port *port)
{
        port->attr.state                = IB_PORT_DOWN;
        port->attr.max_mtu                = IB_MTU_4096;
        port->attr.active_mtu                = IB_MTU_256;
        port->attr.gid_tbl_len                = RXE_PORT_GID_TBL_LEN;
        port->attr.port_cap_flags        = RXE_PORT_PORT_CAP_FLAGS;
        port->attr.max_msg_sz                = RXE_PORT_MAX_MSG_SZ;
        port->attr.bad_pkey_cntr        = RXE_PORT_BAD_PKEY_CNTR;
        port->attr.qkey_viol_cntr        = RXE_PORT_QKEY_VIOL_CNTR;
        port->attr.pkey_tbl_len                = RXE_PORT_PKEY_TBL_LEN;
        port->attr.lid                        = RXE_PORT_LID;
        port->attr.sm_lid                = RXE_PORT_SM_LID;
        port->attr.lmc                        = RXE_PORT_LMC;
        port->attr.max_vl_num                = RXE_PORT_MAX_VL_NUM;
        port->attr.sm_sl                = RXE_PORT_SM_SL;
        port->attr.subnet_timeout        = RXE_PORT_SUBNET_TIMEOUT;
        port->attr.init_type_reply        = RXE_PORT_INIT_TYPE_REPLY;
        port->attr.active_width                = RXE_PORT_ACTIVE_WIDTH;
        port->attr.active_speed                = RXE_PORT_ACTIVE_SPEED;
        port->attr.phys_state                = RXE_PORT_PHYS_STATE;
        port->mtu_cap                        = ib_mtu_enum_to_int(IB_MTU_256);
        port->subnet_prefix                = cpu_to_be64(RXE_PORT_SUBNET_PREFIX);
}

/* initialize port state, note IB convention that HCA ports are always
 * numbered from 1
 */
static void rxe_init_ports(struct rxe_dev *rxe, struct net_device *ndev)
{
        struct rxe_port *port = &rxe->port;

        rxe_init_port_param(port);
        addrconf_addr_eui48((unsigned char *)&port->port_guid,
                            rxe->raw_gid);
        spin_lock_init(&port->port_lock);
}

/* init pools of managed objects */
static void rxe_init_pools(struct rxe_dev *rxe)
{
        rxe_pool_init(rxe, &rxe->uc_pool, RXE_TYPE_UC);
        rxe_pool_init(rxe, &rxe->pd_pool, RXE_TYPE_PD);
        rxe_pool_init(rxe, &rxe->ah_pool, RXE_TYPE_AH);
        rxe_pool_init(rxe, &rxe->srq_pool, RXE_TYPE_SRQ);
        rxe_pool_init(rxe, &rxe->qp_pool, RXE_TYPE_QP);
        rxe_pool_init(rxe, &rxe->cq_pool, RXE_TYPE_CQ);
        rxe_pool_init(rxe, &rxe->mr_pool, RXE_TYPE_MR);
        rxe_pool_init(rxe, &rxe->mw_pool, RXE_TYPE_MW);
}

/* initialize rxe device state */
static void rxe_init(struct rxe_dev *rxe, struct net_device *ndev)
{
        /* init default device parameters */
        rxe_init_device_param(rxe, ndev);

        rxe_init_ports(rxe, ndev);
        rxe_init_pools(rxe);

        /* init pending mmap list */
        spin_lock_init(&rxe->mmap_offset_lock);
        spin_lock_init(&rxe->pending_lock);
        INIT_LIST_HEAD(&rxe->pending_mmaps);

        /* init multicast support */
        spin_lock_init(&rxe->mcg_lock);
        rxe->mcg_tree = RB_ROOT;

        mutex_init(&rxe->usdev_lock);
}

void rxe_set_mtu(struct rxe_dev *rxe, unsigned int ndev_mtu)
{
        struct rxe_port *port = &rxe->port;
        enum ib_mtu mtu;

        mtu = eth_mtu_int_to_enum(ndev_mtu);

        /* Make sure that new MTU in range */
        mtu = mtu ? min_t(enum ib_mtu, mtu, IB_MTU_4096) : IB_MTU_256;

        port->attr.active_mtu = mtu;
        port->mtu_cap = ib_mtu_enum_to_int(mtu);
}

static struct rdma_link_ops rxe_link_ops;

/* called by ifc layer to create new rxe device.
 * The caller should allocate memory for rxe by calling ib_alloc_device.
 */
int rxe_add(struct rxe_dev *rxe, unsigned int mtu, const char *ibdev_name,
                        struct net_device *ndev)
{
        rxe_init(rxe, ndev);
        rxe_set_mtu(rxe, mtu);
        rxe->ib_dev.link_ops = &rxe_link_ops;

        return rxe_register_device(rxe, ibdev_name, ndev);
}

static int rxe_newlink(const char *ibdev_name, struct net_device *ndev)
{
        struct rxe_dev *rxe;
        int err = 0;

        if (is_vlan_dev(ndev)) {
                rxe_err("rxe creation allowed on top of a real device only\n");
                err = -EPERM;
                goto err;
        }

        rxe = rxe_get_dev_from_net(ndev);
        if (rxe) {
                ib_device_put(&rxe->ib_dev);
                rxe_err_dev(rxe, "already configured on %s\n", ndev->name);
                err = -EEXIST;
                goto err;
        }

        err = rxe_net_init(ndev);
        if (err)
                return err;

        err = rxe_net_add(ibdev_name, ndev);
        if (err) {
                rxe_err("failed to add %s\n", ndev->name);
                goto err;
        }
err:
        return err;
}

static int rxe_dellink(struct ib_device *dev)
{
        rxe_net_del(dev);

        return 0;
}

static struct rdma_link_ops rxe_link_ops = {
        .type = "rxe",
        .newlink = rxe_newlink,
        .dellink = rxe_dellink,
};

static int __init rxe_module_init(void)
{
        int err;

        err = rxe_alloc_wq();
        if (err)
                return err;

        err = rxe_namespace_init();
        if (err)
                goto err_destroy_wq;

        err = rxe_register_notifier();
        if (err)
                goto err_namespace_exit;

        rdma_link_register(&rxe_link_ops);

        pr_info("loaded\n");
        return 0;

err_namespace_exit:
        rxe_namespace_exit();
err_destroy_wq:
        rxe_destroy_wq();
        return err;
}

static void __exit rxe_module_exit(void)
{
        rdma_link_unregister(&rxe_link_ops);
        ib_unregister_driver(RDMA_DRIVER_RXE);
        rxe_net_exit();
        rxe_destroy_wq();

        rxe_namespace_exit();

        pr_info("unloaded\n");
}

late_initcall(rxe_module_init);
module_exit(rxe_module_exit);

MODULE_ALIAS_RDMA_LINK("rxe");





























 4697 







 4692 





 4697 




 11997 





 11987 





 4693 











 4696 



 4566 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * printk_safe.c - Safe printk for printk-deadlock-prone contexts
 */

#include <linux/preempt.h>
#include <linux/kdb.h>
#include <linux/smp.h>
#include <linux/cpumask.h>
#include <linux/printk.h>
#include <linux/kprobes.h>

#include "internal.h"

/* Context where printk messages are never suppressed */
static atomic_t force_con;

void printk_force_console_enter(void)
{
        atomic_inc(&force_con);
}

void printk_force_console_exit(void)
{
        atomic_dec(&force_con);
}

bool is_printk_force_console(void)
{
        return atomic_read(&force_con);
}

static DEFINE_PER_CPU(int, printk_context);

/* Can be preempted by NMI. */
void __printk_safe_enter(void)
{
        this_cpu_inc(printk_context);
}

/* Can be preempted by NMI. */
void __printk_safe_exit(void)
{
        this_cpu_dec(printk_context);
}

void __printk_deferred_enter(void)
{
        cant_migrate();
        __printk_safe_enter();
}

void __printk_deferred_exit(void)
{
        cant_migrate();
        __printk_safe_exit();
}

bool is_printk_legacy_deferred(void)
{
        /*
         * The per-CPU variable @printk_context can be read safely in any
         * context. CPU migration is always disabled when set.
         *
         * A context holding the printk_cpu_sync must not spin waiting for
         * another CPU. For legacy printing, it could be the console_lock
         * or the port lock.
         */
        return (force_legacy_kthread() ||
                this_cpu_read(printk_context) ||
                in_nmi() ||
                is_printk_cpu_sync_owner());
}

asmlinkage int vprintk(const char *fmt, va_list args)
{
#ifdef CONFIG_KGDB_KDB
        /* Allow to pass printk() to kdb but avoid a recursion. */
        if (unlikely(kdb_trap_printk && kdb_printf_cpu < 0))
                return vkdb_printf(KDB_MSGSRC_PRINTK, fmt, args);
#endif
        return vprintk_default(fmt, args);
}
EXPORT_SYMBOL(vprintk);









































    2 



    2 











































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * sctp_offload - GRO/GSO Offloading for SCTP
 *
 * Copyright (C) 2015, Marcelo Ricardo Leitner <marcelo.leitner@gmail.com>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/kprobes.h>
#include <linux/socket.h>
#include <linux/sctp.h>
#include <linux/proc_fs.h>
#include <linux/vmalloc.h>
#include <linux/module.h>
#include <linux/kfifo.h>
#include <linux/time.h>
#include <net/net_namespace.h>

#include <linux/skbuff.h>
#include <net/sctp/sctp.h>
#include <net/sctp/checksum.h>
#include <net/protocol.h>
#include <net/gso.h>

static __le32 sctp_gso_make_checksum(struct sk_buff *skb)
{
        skb->ip_summed = CHECKSUM_NONE;
        skb->csum_not_inet = 0;
        /* csum and csum_start in GSO CB may be needed to do the UDP
         * checksum when it's a UDP tunneling packet.
         */
        SKB_GSO_CB(skb)->csum = (__force __wsum)~0;
        SKB_GSO_CB(skb)->csum_start = skb_headroom(skb) + skb->len;
        return sctp_compute_cksum(skb, skb_transport_offset(skb));
}

static struct sk_buff *sctp_gso_segment(struct sk_buff *skb,
                                        netdev_features_t features)
{
        struct sk_buff *segs = ERR_PTR(-EINVAL);
        struct sctphdr *sh;

        if (!skb_is_gso_sctp(skb))
                goto out;

        sh = sctp_hdr(skb);
        if (!pskb_may_pull(skb, sizeof(*sh)))
                goto out;

        __skb_pull(skb, sizeof(*sh));

        if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {
                /* Packet is from an untrusted source, reset gso_segs. */
                struct skb_shared_info *pinfo = skb_shinfo(skb);
                struct sk_buff *frag_iter;

                pinfo->gso_segs = 0;
                if (skb->len != skb->data_len) {
                        /* Means we have chunks in here too */
                        pinfo->gso_segs++;
                }

                skb_walk_frags(skb, frag_iter)
                        pinfo->gso_segs++;

                segs = NULL;
                goto out;
        }

        segs = skb_segment(skb, (features | NETIF_F_HW_CSUM) & ~NETIF_F_SG);
        if (IS_ERR(segs))
                goto out;

        /* All that is left is update SCTP CRC if necessary */
        if (!(features & NETIF_F_SCTP_CRC)) {
                for (skb = segs; skb; skb = skb->next) {
                        if (skb->ip_summed == CHECKSUM_PARTIAL) {
                                sh = sctp_hdr(skb);
                                sh->checksum = sctp_gso_make_checksum(skb);
                        }
                }
        }

out:
        return segs;
}

static const struct net_offload sctp_offload = {
        .callbacks = {
                .gso_segment = sctp_gso_segment,
        },
};

static const struct net_offload sctp6_offload = {
        .callbacks = {
                .gso_segment = sctp_gso_segment,
        },
};

int __init sctp_offload_init(void)
{
        int ret;

        ret = inet_add_offload(&sctp_offload, IPPROTO_SCTP);
        if (ret)
                goto out;

        ret = inet6_add_offload(&sctp6_offload, IPPROTO_SCTP);
        if (ret)
                goto ipv4;

        return ret;

ipv4:
        inet_del_offload(&sctp_offload, IPPROTO_SCTP);
out:
        return ret;
}































































































































    6 





























































































































 10978 





  930 

























































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_NODEMASK_H
#define __LINUX_NODEMASK_H

/*
 * Nodemasks provide a bitmap suitable for representing the
 * set of Node's in a system, one bit position per Node number.
 *
 * See detailed comments in the file linux/bitmap.h describing the
 * data type on which these nodemasks are based.
 *
 * For details of nodemask_parse_user(), see bitmap_parse_user() in
 * lib/bitmap.c.  For details of nodelist_parse(), see bitmap_parselist(),
 * also in bitmap.c.  For details of node_remap(), see bitmap_bitremap in
 * lib/bitmap.c.  For details of nodes_remap(), see bitmap_remap in
 * lib/bitmap.c.  For details of nodes_onto(), see bitmap_onto in
 * lib/bitmap.c.  For details of nodes_fold(), see bitmap_fold in
 * lib/bitmap.c.
 *
 * The available nodemask operations are:
 *
 * void node_set(node, mask)                turn on bit 'node' in mask
 * void node_clear(node, mask)                turn off bit 'node' in mask
 * void nodes_setall(mask)                set all bits
 * void nodes_clear(mask)                clear all bits
 * int node_isset(node, mask)                true iff bit 'node' set in mask
 * int node_test_and_set(node, mask)        test and set bit 'node' in mask
 *
 * void nodes_and(dst, src1, src2)        dst = src1 & src2  [intersection]
 * void nodes_or(dst, src1, src2)        dst = src1 | src2  [union]
 * void nodes_xor(dst, src1, src2)        dst = src1 ^ src2
 * void nodes_andnot(dst, src1, src2)        dst = src1 & ~src2
 * void nodes_complement(dst, src)        dst = ~src
 *
 * int nodes_equal(mask1, mask2)        Does mask1 == mask2?
 * int nodes_intersects(mask1, mask2)        Do mask1 and mask2 intersect?
 * int nodes_subset(mask1, mask2)        Is mask1 a subset of mask2?
 * int nodes_empty(mask)                Is mask empty (no bits sets)?
 * int nodes_full(mask)                        Is mask full (all bits sets)?
 * int nodes_weight(mask)                Hamming weight - number of set bits
 *
 * unsigned int first_node(mask)        Number lowest set bit, or MAX_NUMNODES
 * unsigend int next_node(node, mask)        Next node past 'node', or MAX_NUMNODES
 * unsigned int next_node_in(node, mask) Next node past 'node', or wrap to first,
 *                                        or MAX_NUMNODES
 * unsigned int first_unset_node(mask)        First node not set in mask, or
 *                                        MAX_NUMNODES
 *
 * nodemask_t nodemask_of_node(node)        Return nodemask with bit 'node' set
 * NODE_MASK_ALL                        Initializer - all bits set
 * NODE_MASK_NONE                        Initializer - no bits set
 * unsigned long *nodes_addr(mask)        Array of unsigned long's in mask
 *
 * int nodemask_parse_user(ubuf, ulen, mask)        Parse ascii string as nodemask
 * int nodelist_parse(buf, map)                Parse ascii string as nodelist
 * int node_remap(oldbit, old, new)        newbit = map(old, new)(oldbit)
 * void nodes_remap(dst, src, old, new)        *dst = map(old, new)(src)
 * void nodes_onto(dst, orig, relmap)        *dst = orig relative to relmap
 * void nodes_fold(dst, orig, sz)        dst bits = orig bits mod sz
 *
 * for_each_node_mask(node, mask)        for-loop node over mask
 *
 * int num_online_nodes()                Number of online Nodes
 * int num_possible_nodes()                Number of all possible Nodes
 *
 * int node_random(mask)                Random node with set bit in mask
 *
 * int node_online(node)                Is some node online?
 * int node_possible(node)                Is some node possible?
 *
 * node_set_online(node)                set bit 'node' in node_online_map
 * node_set_offline(node)                clear bit 'node' in node_online_map
 *
 * for_each_node(node)                        for-loop node over node_possible_map
 * for_each_online_node(node)                for-loop node over node_online_map
 *
 * Subtlety:
 * 1) The 'type-checked' form of node_isset() causes gcc (3.3.2, anyway)
 *    to generate slightly worse code.  So use a simple one-line #define
 *    for node_isset(), instead of wrapping an inline inside a macro, the
 *    way we do the other calls.
 *
 * NODEMASK_SCRATCH
 * When doing above logical AND, OR, XOR, Remap operations the callers tend to
 * need temporary nodemask_t's on the stack. But if NODES_SHIFT is large,
 * nodemask_t's consume too much stack space.  NODEMASK_SCRATCH is a helper
 * for such situations. See below and CPUMASK_ALLOC also.
 */

#include <linux/threads.h>
#include <linux/bitmap.h>
#include <linux/minmax.h>
#include <linux/nodemask_types.h>
#include <linux/random.h>

extern nodemask_t _unused_nodemask_arg_;

/**
 * nodemask_pr_args - printf args to output a nodemask
 * @maskp: nodemask to be printed
 *
 * Can be used to provide arguments for '%*pb[l]' when printing a nodemask.
 */
#define nodemask_pr_args(maskp)        __nodemask_pr_numnodes(maskp), \
                                __nodemask_pr_bits(maskp)
static __always_inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
{
        return m ? MAX_NUMNODES : 0;
}
static __always_inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)
{
        return m ? m->bits : NULL;
}

/*
 * The inline keyword gives the compiler room to decide to inline, or
 * not inline a function as it sees best.  However, as these functions
 * are called in both __init and non-__init functions, if they are not
 * inlined we will end up with a section mismatch error (of the type of
 * freeable items not being freed).  So we must use __always_inline here
 * to fix the problem.  If other functions in the future also end up in
 * this situation they will also need to be annotated as __always_inline
 */
#define node_set(node, dst) __node_set((node), &(dst))
static __always_inline void __node_set(int node, volatile nodemask_t *dstp)
{
        set_bit(node, dstp->bits);
}

#define node_clear(node, dst) __node_clear((node), &(dst))
static __always_inline void __node_clear(int node, volatile nodemask_t *dstp)
{
        clear_bit(node, dstp->bits);
}

#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
static __always_inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
{
        bitmap_fill(dstp->bits, nbits);
}

#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
static __always_inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
{
        bitmap_zero(dstp->bits, nbits);
}

/* No static inline type checking - see Subtlety (1) above. */
#define node_isset(node, nodemask) test_bit((node), (nodemask).bits)

#define node_test_and_set(node, nodemask) \
                        __node_test_and_set((node), &(nodemask))
static __always_inline bool __node_test_and_set(int node, nodemask_t *addr)
{
        return test_and_set_bit(node, addr->bits);
}

#define nodes_and(dst, src1, src2) \
                        __nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
static __always_inline bool __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
                                        const nodemask_t *src2p, unsigned int nbits)
{
        return bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_or(dst, src1, src2) \
                        __nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
static __always_inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
                                        const nodemask_t *src2p, unsigned int nbits)
{
        bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_xor(dst, src1, src2) \
                        __nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
static __always_inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
                                        const nodemask_t *src2p, unsigned int nbits)
{
        bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_andnot(dst, src1, src2) \
                        __nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
static __always_inline bool __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
                                        const nodemask_t *src2p, unsigned int nbits)
{
        return bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
}

#define nodes_copy(dst, src) __nodes_copy(&(dst), &(src), MAX_NUMNODES)
static __always_inline void __nodes_copy(nodemask_t *dstp,
                                        const nodemask_t *srcp, unsigned int nbits)
{
        bitmap_copy(dstp->bits, srcp->bits, nbits);
}

#define nodes_complement(dst, src) \
                        __nodes_complement(&(dst), &(src), MAX_NUMNODES)
static __always_inline void __nodes_complement(nodemask_t *dstp,
                                        const nodemask_t *srcp, unsigned int nbits)
{
        bitmap_complement(dstp->bits, srcp->bits, nbits);
}

#define nodes_equal(src1, src2) \
                        __nodes_equal(&(src1), &(src2), MAX_NUMNODES)
static __always_inline bool __nodes_equal(const nodemask_t *src1p,
                                        const nodemask_t *src2p, unsigned int nbits)
{
        return bitmap_equal(src1p->bits, src2p->bits, nbits);
}

#define nodes_intersects(src1, src2) \
                        __nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
static __always_inline bool __nodes_intersects(const nodemask_t *src1p,
                                        const nodemask_t *src2p, unsigned int nbits)
{
        return bitmap_intersects(src1p->bits, src2p->bits, nbits);
}

#define nodes_subset(src1, src2) \
                        __nodes_subset(&(src1), &(src2), MAX_NUMNODES)
static __always_inline bool __nodes_subset(const nodemask_t *src1p,
                                        const nodemask_t *src2p, unsigned int nbits)
{
        return bitmap_subset(src1p->bits, src2p->bits, nbits);
}

#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
static __always_inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
{
        return bitmap_empty(srcp->bits, nbits);
}

#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
static __always_inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)
{
        return bitmap_full(srcp->bits, nbits);
}

#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
static __always_inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
{
        return bitmap_weight(srcp->bits, nbits);
}

/* FIXME: better would be to fix all architectures to never return
          > MAX_NUMNODES, then the silly min()s could be dropped. */

#define first_node(src) __first_node(&(src))
static __always_inline unsigned int __first_node(const nodemask_t *srcp)
{
        return min(MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
}

#define next_node(n, src) __next_node((n), &(src))
static __always_inline unsigned int __next_node(int n, const nodemask_t *srcp)
{
        return min(MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
}

/*
 * Find the next present node in src, starting after node n, wrapping around to
 * the first node in src if needed.  Returns MAX_NUMNODES if src is empty.
 */
#define next_node_in(n, src) __next_node_in((n), &(src))
static __always_inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
{
        unsigned int ret = __next_node(node, srcp);

        if (ret == MAX_NUMNODES)
                ret = __first_node(srcp);
        return ret;
}

static __always_inline void init_nodemask_of_node(nodemask_t *mask, int node)
{
        nodes_clear(*mask);
        node_set(node, *mask);
}

#define nodemask_of_node(node)                                                \
({                                                                        \
        typeof(_unused_nodemask_arg_) m;                                \
        if (sizeof(m) == sizeof(unsigned long)) {                        \
                m.bits[0] = 1UL << (node);                                \
        } else {                                                        \
                init_nodemask_of_node(&m, (node));                        \
        }                                                                \
        m;                                                                \
})

#define first_unset_node(mask) __first_unset_node(&(mask))
static __always_inline unsigned int __first_unset_node(const nodemask_t *maskp)
{
        return min(MAX_NUMNODES, find_first_zero_bit(maskp->bits, MAX_NUMNODES));
}

#define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES)

#if MAX_NUMNODES <= BITS_PER_LONG

#define NODE_MASK_ALL                                                        \
((nodemask_t) { {                                                        \
        [BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD                \
} })

#else

#define NODE_MASK_ALL                                                        \
((nodemask_t) { {                                                        \
        [0 ... BITS_TO_LONGS(MAX_NUMNODES)-2] = ~0UL,                        \
        [BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD                \
} })

#endif

#define NODE_MASK_NONE                                                        \
((nodemask_t) { {                                                        \
        [0 ... BITS_TO_LONGS(MAX_NUMNODES)-1] =  0UL                        \
} })

#define nodes_addr(src) ((src).bits)

#define nodemask_parse_user(ubuf, ulen, dst) \
                __nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
static __always_inline int __nodemask_parse_user(const char __user *buf, int len,
                                        nodemask_t *dstp, int nbits)
{
        return bitmap_parse_user(buf, len, dstp->bits, nbits);
}

#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
static __always_inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)
{
        return bitmap_parselist(buf, dstp->bits, nbits);
}

#define node_remap(oldbit, old, new) \
                __node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
static __always_inline int __node_remap(int oldbit,
                const nodemask_t *oldp, const nodemask_t *newp, int nbits)
{
        return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
}

#define nodes_remap(dst, src, old, new) \
                __nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
static __always_inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
                const nodemask_t *oldp, const nodemask_t *newp, int nbits)
{
        bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
}

#define nodes_onto(dst, orig, relmap) \
                __nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
static __always_inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
                const nodemask_t *relmapp, int nbits)
{
        bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
}

#define nodes_fold(dst, orig, sz) \
                __nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
static __always_inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,
                int sz, int nbits)
{
        bitmap_fold(dstp->bits, origp->bits, sz, nbits);
}

#if MAX_NUMNODES > 1
#define for_each_node_mask(node, mask)                                    \
        for ((node) = first_node(mask);                                    \
             (node) < MAX_NUMNODES;                                    \
             (node) = next_node((node), (mask)))
#else /* MAX_NUMNODES == 1 */
#define for_each_node_mask(node, mask)                                  \
        for ((node) = 0; (node) < 1 && !nodes_empty(mask); (node)++)
#endif /* MAX_NUMNODES */

/*
 * Bitmasks that are kept for all the nodes.
 */
enum node_states {
        N_POSSIBLE,                /* The node could become online at some point */
        N_ONLINE,                /* The node is online */
        N_NORMAL_MEMORY,        /* The node has regular memory */
#ifdef CONFIG_HIGHMEM
        N_HIGH_MEMORY,                /* The node has regular or high memory */
#else
        N_HIGH_MEMORY = N_NORMAL_MEMORY,
#endif
        N_MEMORY,                /* The node has memory(regular, high, movable) */
        N_CPU,                /* The node has one or more cpus */
        N_GENERIC_INITIATOR,        /* The node has one or more Generic Initiators */
        NR_NODE_STATES
};

/*
 * The following particular system nodemasks and operations
 * on them manage all possible and online nodes.
 */

extern nodemask_t node_states[NR_NODE_STATES];

#if MAX_NUMNODES > 1
static __always_inline int node_state(int node, enum node_states state)
{
        return node_isset(node, node_states[state]);
}

static __always_inline void node_set_state(int node, enum node_states state)
{
        __node_set(node, &node_states[state]);
}

static __always_inline void node_clear_state(int node, enum node_states state)
{
        __node_clear(node, &node_states[state]);
}

static __always_inline int num_node_state(enum node_states state)
{
        return nodes_weight(node_states[state]);
}

#define for_each_node_state(__node, __state) \
        for_each_node_mask((__node), node_states[__state])

#define first_online_node        first_node(node_states[N_ONLINE])
#define first_memory_node        first_node(node_states[N_MEMORY])
static __always_inline unsigned int next_online_node(int nid)
{
        return next_node(nid, node_states[N_ONLINE]);
}
static __always_inline unsigned int next_memory_node(int nid)
{
        return next_node(nid, node_states[N_MEMORY]);
}

extern unsigned int nr_node_ids;
extern unsigned int nr_online_nodes;

static __always_inline void node_set_online(int nid)
{
        node_set_state(nid, N_ONLINE);
        nr_online_nodes = num_node_state(N_ONLINE);
}

static __always_inline void node_set_offline(int nid)
{
        node_clear_state(nid, N_ONLINE);
        nr_online_nodes = num_node_state(N_ONLINE);
}

#else

static __always_inline int node_state(int node, enum node_states state)
{
        return node == 0;
}

static __always_inline void node_set_state(int node, enum node_states state)
{
}

static __always_inline void node_clear_state(int node, enum node_states state)
{
}

static __always_inline int num_node_state(enum node_states state)
{
        return 1;
}

#define for_each_node_state(node, __state) \
        for ( (node) = 0; (node) == 0; (node) = 1)

#define first_online_node        0
#define first_memory_node        0
#define next_online_node(nid)        (MAX_NUMNODES)
#define next_memory_node(nid)        (MAX_NUMNODES)
#define nr_node_ids                1U
#define nr_online_nodes                1U

#define node_set_online(node)           node_set_state((node), N_ONLINE)
#define node_set_offline(node)           node_clear_state((node), N_ONLINE)

#endif

static __always_inline int node_random(const nodemask_t *maskp)
{
#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
        int node = find_random_bit(maskp->bits, MAX_NUMNODES);

        return node < MAX_NUMNODES ? node : NUMA_NO_NODE;
#else
        return 0;
#endif
}

#define node_online_map         node_states[N_ONLINE]
#define node_possible_map         node_states[N_POSSIBLE]

#define num_online_nodes()        num_node_state(N_ONLINE)
#define num_possible_nodes()        num_node_state(N_POSSIBLE)
#define node_online(node)        node_state((node), N_ONLINE)
#define node_possible(node)        node_state((node), N_POSSIBLE)

#define for_each_node(node)           for_each_node_state(node, N_POSSIBLE)
#define for_each_online_node(node) for_each_node_state(node, N_ONLINE)
#define for_each_node_with_cpus(node)        for_each_node_state(node, N_CPU)

/*
 * For nodemask scratch area.
 * NODEMASK_ALLOC(type, name) allocates an object with a specified type and
 * name.
 */
#if NODES_SHIFT > 8 /* nodemask_t > 32 bytes */
#define NODEMASK_ALLOC(type, name, gfp_flags)        \
                        type *name = kmalloc(sizeof(*name), gfp_flags)
#define NODEMASK_FREE(m)                        kfree(m)
#else
#define NODEMASK_ALLOC(type, name, gfp_flags)        type _##name, *name = &_##name
#define NODEMASK_FREE(m)                        do {} while (0)
#endif

/* Example structure for using NODEMASK_ALLOC, used in mempolicy. */
struct nodemask_scratch {
        nodemask_t        mask1;
        nodemask_t        mask2;
};

#define NODEMASK_SCRATCH(x)                                                \
                        NODEMASK_ALLOC(struct nodemask_scratch, x,        \
                                        GFP_KERNEL | __GFP_NORETRY)
#define NODEMASK_SCRATCH_FREE(x)        NODEMASK_FREE(x)


#endif /* __LINUX_NODEMASK_H */





















   69 


















    2 




   68 













1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM msr

#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE msr-trace

#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH asm/

#if !defined(_TRACE_MSR_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_MSR_H

#include <linux/tracepoint.h>

/*
 * Tracing for x86 model specific registers. Directly maps to the
 * RDMSR/WRMSR instructions.
 */

DECLARE_EVENT_CLASS(msr_trace_class,
            TP_PROTO(unsigned msr, u64 val, int failed),
            TP_ARGS(msr, val, failed),
            TP_STRUCT__entry(
                    __field(        unsigned,        msr )
                    __field(    u64,                val )
                    __field(    int,                failed )
            ),
            TP_fast_assign(
                    __entry->msr = msr;
                    __entry->val = val;
                    __entry->failed = failed;
            ),
            TP_printk("%x, value %llx%s",
                      __entry->msr,
                      __entry->val,
                      __entry->failed ? " #GP" : "")
);

DEFINE_EVENT(msr_trace_class, read_msr,
             TP_PROTO(unsigned msr, u64 val, int failed),
             TP_ARGS(msr, val, failed)
);

DEFINE_EVENT(msr_trace_class, write_msr,
             TP_PROTO(unsigned msr, u64 val, int failed),
             TP_ARGS(msr, val, failed)
);

DEFINE_EVENT(msr_trace_class, rdpmc,
             TP_PROTO(unsigned msr, u64 val, int failed),
             TP_ARGS(msr, val, failed)
);

#endif /* _TRACE_MSR_H */

/* This part must be outside protection */
#include <trace/define_trace.h>




























 6260 
  175 














 12167 
  295 































































































 14274 
  561 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
/* SPDX-License-Identifier: GPL-2.0 */

/*
 * This file provides wrappers with sanitizer instrumentation for non-atomic
 * bit operations.
 *
 * To use this functionality, an arch's bitops.h file needs to define each of
 * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
 * arch___set_bit(), etc.).
 */
#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
#define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H

#include <linux/instrumented.h>

/**
 * ___set_bit - Set a bit in memory
 * @nr: the bit to set
 * @addr: the address to start counting from
 *
 * Unlike set_bit(), this function is non-atomic. If it is called on the same
 * region of memory concurrently, the effect may be that only one operation
 * succeeds.
 */
static __always_inline void
___set_bit(unsigned long nr, volatile unsigned long *addr)
{
        instrument_write(addr + BIT_WORD(nr), sizeof(long));
        arch___set_bit(nr, addr);
}

/**
 * ___clear_bit - Clears a bit in memory
 * @nr: the bit to clear
 * @addr: the address to start counting from
 *
 * Unlike clear_bit(), this function is non-atomic. If it is called on the same
 * region of memory concurrently, the effect may be that only one operation
 * succeeds.
 */
static __always_inline void
___clear_bit(unsigned long nr, volatile unsigned long *addr)
{
        instrument_write(addr + BIT_WORD(nr), sizeof(long));
        arch___clear_bit(nr, addr);
}

/**
 * ___change_bit - Toggle a bit in memory
 * @nr: the bit to change
 * @addr: the address to start counting from
 *
 * Unlike change_bit(), this function is non-atomic. If it is called on the same
 * region of memory concurrently, the effect may be that only one operation
 * succeeds.
 */
static __always_inline void
___change_bit(unsigned long nr, volatile unsigned long *addr)
{
        instrument_write(addr + BIT_WORD(nr), sizeof(long));
        arch___change_bit(nr, addr);
}

static __always_inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
{
        if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
                /*
                 * We treat non-atomic read-write bitops a little more special.
                 * Given the operations here only modify a single bit, assuming
                 * non-atomicity of the writer is sufficient may be reasonable
                 * for certain usage (and follows the permissible nature of the
                 * assume-plain-writes-atomic rule):
                 * 1. report read-modify-write races -> check read;
                 * 2. do not report races with marked readers, but do report
                 *    races with unmarked readers -> check "atomic" write.
                 */
                kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
                /*
                 * Use generic write instrumentation, in case other sanitizers
                 * or tools are enabled alongside KCSAN.
                 */
                instrument_write(addr + BIT_WORD(nr), sizeof(long));
        } else {
                instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
        }
}

/**
 * ___test_and_set_bit - Set a bit and return its old value
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This operation is non-atomic. If two instances of this operation race, one
 * can appear to succeed but actually fail.
 */
static __always_inline bool
___test_and_set_bit(unsigned long nr, volatile unsigned long *addr)
{
        __instrument_read_write_bitop(nr, addr);
        return arch___test_and_set_bit(nr, addr);
}

/**
 * ___test_and_clear_bit - Clear a bit and return its old value
 * @nr: Bit to clear
 * @addr: Address to count from
 *
 * This operation is non-atomic. If two instances of this operation race, one
 * can appear to succeed but actually fail.
 */
static __always_inline bool
___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr)
{
        __instrument_read_write_bitop(nr, addr);
        return arch___test_and_clear_bit(nr, addr);
}

/**
 * ___test_and_change_bit - Change a bit and return its old value
 * @nr: Bit to change
 * @addr: Address to count from
 *
 * This operation is non-atomic. If two instances of this operation race, one
 * can appear to succeed but actually fail.
 */
static __always_inline bool
___test_and_change_bit(unsigned long nr, volatile unsigned long *addr)
{
        __instrument_read_write_bitop(nr, addr);
        return arch___test_and_change_bit(nr, addr);
}

/**
 * _test_bit - Determine whether a bit is set
 * @nr: bit number to test
 * @addr: Address to start counting from
 */
static __always_inline bool
_test_bit(unsigned long nr, const volatile unsigned long *addr)
{
        instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
        return arch_test_bit(nr, addr);
}

/**
 * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set
 * @nr: bit number to test
 * @addr: Address to start counting from
 */
static __always_inline bool
_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
{
        instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
        return arch_test_bit_acquire(nr, addr);
}

#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */










































































































































































































  156 




   85 




  300 




   75 







   59 
   79 



























   16 




   23 





































































   17 

























































  121 























   30 





































































































































  205 


  205 

  160 





   58 


   58 


   16 






































































  300 
























   10 































































































































































































































































































































































   54 


































































































































































































   87 
   86 





















  326 
















   24 




   66 
   24 




   66 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
/*
 * net/tipc/msg.h: Include file for TIPC message header routines
 *
 * Copyright (c) 2000-2007, 2014-2017 Ericsson AB
 * Copyright (c) 2005-2008, 2010-2011, Wind River Systems
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _TIPC_MSG_H
#define _TIPC_MSG_H

#include <linux/tipc.h>
#include "core.h"

/*
 * Constants and routines used to read and write TIPC payload message headers
 *
 * Note: Some items are also used with TIPC internal message headers
 */
#define TIPC_VERSION              2
struct plist;

/*
 * Payload message users are defined in TIPC's public API:
 * - TIPC_LOW_IMPORTANCE
 * - TIPC_MEDIUM_IMPORTANCE
 * - TIPC_HIGH_IMPORTANCE
 * - TIPC_CRITICAL_IMPORTANCE
 */
#define TIPC_SYSTEM_IMPORTANCE        4


/*
 * Payload message types
 */
#define TIPC_CONN_MSG           0
#define TIPC_MCAST_MSG          1
#define TIPC_NAMED_MSG          2
#define TIPC_DIRECT_MSG         3
#define TIPC_GRP_MEMBER_EVT     4
#define TIPC_GRP_BCAST_MSG      5
#define TIPC_GRP_MCAST_MSG      6
#define TIPC_GRP_UCAST_MSG      7

/*
 * Internal message users
 */
#define  BCAST_PROTOCOL       5
#define  MSG_BUNDLER          6
#define  LINK_PROTOCOL        7
#define  CONN_MANAGER         8
#define  GROUP_PROTOCOL       9
#define  TUNNEL_PROTOCOL      10
#define  NAME_DISTRIBUTOR     11
#define  MSG_FRAGMENTER       12
#define  LINK_CONFIG          13
#define  MSG_CRYPTO           14
#define  SOCK_WAKEUP          14       /* pseudo user */
#define  TOP_SRV              15       /* pseudo user */

/*
 * Message header sizes
 */
#define SHORT_H_SIZE              24        /* In-cluster basic payload message */
#define BASIC_H_SIZE              32        /* Basic payload message */
#define NAMED_H_SIZE              40        /* Named payload message */
#define MCAST_H_SIZE              44        /* Multicast payload message */
#define GROUP_H_SIZE              44        /* Group payload message */
#define INT_H_SIZE                40        /* Internal messages */
#define MIN_H_SIZE                24        /* Smallest legal TIPC header size */
#define MAX_H_SIZE                60        /* Largest possible TIPC header size */

#define MAX_MSG_SIZE (MAX_H_SIZE + TIPC_MAX_USER_MSG_SIZE)
#define TIPC_MEDIA_INFO_OFFSET        5

extern const int one_page_mtu;

struct tipc_skb_cb {
        union {
                struct {
                        struct sk_buff *tail;
                        unsigned long nxt_retr;
                        unsigned long retr_stamp;
                        u32 bytes_read;
                        u32 orig_member;
                        u16 chain_imp;
                        u16 ackers;
                        u16 retr_cnt;
                } __packed;
#ifdef CONFIG_TIPC_CRYPTO
                struct {
                        struct tipc_crypto *rx;
                        struct tipc_aead *last;
                        u8 recurs;
                } tx_clone_ctx __packed;
#endif
        } __packed;
        union {
                struct {
                        u8 validated:1;
#ifdef CONFIG_TIPC_CRYPTO
                        u8 encrypted:1;
                        u8 decrypted:1;
#define SKB_PROBING        1
#define SKB_GRACING        2
                        u8 xmit_type:2;
                        u8 tx_clone_deferred:1;
#endif
                };
                u8 flags;
        };
        u8 reserved;
#ifdef CONFIG_TIPC_CRYPTO
        void *crypto_ctx;
#endif
} __packed;

#define TIPC_SKB_CB(__skb) ((struct tipc_skb_cb *)&((__skb)->cb[0]))

struct tipc_msg {
        __be32 hdr[15];
};

/* struct tipc_gap_ack - TIPC Gap ACK block
 * @ack: seqno of the last consecutive packet in link deferdq
 * @gap: number of gap packets since the last ack
 *
 * E.g:
 *       link deferdq: 1 2 3 4      10 11      13 14 15       20
 * --> Gap ACK blocks:      <4, 5>,   <11, 1>,      <15, 4>, <20, 0>
 */
struct tipc_gap_ack {
        __be16 ack;
        __be16 gap;
};

/* struct tipc_gap_ack_blks
 * @len: actual length of the record
 * @ugack_cnt: number of Gap ACK blocks for unicast (following the broadcast
 *             ones)
 * @start_index: starting index for "valid" broadcast Gap ACK blocks
 * @bgack_cnt: number of Gap ACK blocks for broadcast in the record
 * @gacks: array of Gap ACK blocks
 *
 *  31                       16 15                        0
 * +-------------+-------------+-------------+-------------+
 * |  bgack_cnt  |  ugack_cnt  |            len            |
 * +-------------+-------------+-------------+-------------+  -
 * |            gap            |            ack            |   |
 * +-------------+-------------+-------------+-------------+    > bc gacks
 * :                           :                           :   |
 * +-------------+-------------+-------------+-------------+  -
 * |            gap            |            ack            |   |
 * +-------------+-------------+-------------+-------------+    > uc gacks
 * :                           :                           :   |
 * +-------------+-------------+-------------+-------------+  -
 */
struct tipc_gap_ack_blks {
        __be16 len;
        union {
                u8 ugack_cnt;
                u8 start_index;
        };
        u8 bgack_cnt;
        struct tipc_gap_ack gacks[];
};

#define MAX_GAP_ACK_BLKS        128
#define MAX_GAP_ACK_BLKS_SZ        (sizeof(struct tipc_gap_ack_blks) + \
                                 sizeof(struct tipc_gap_ack) * MAX_GAP_ACK_BLKS)

static inline struct tipc_msg *buf_msg(struct sk_buff *skb)
{
        return (struct tipc_msg *)skb->data;
}

static inline u32 msg_word(struct tipc_msg *m, u32 pos)
{
        return ntohl(m->hdr[pos]);
}

static inline void msg_set_word(struct tipc_msg *m, u32 w, u32 val)
{
        m->hdr[w] = htonl(val);
}

static inline u32 msg_bits(struct tipc_msg *m, u32 w, u32 pos, u32 mask)
{
        return (msg_word(m, w) >> pos) & mask;
}

static inline void msg_set_bits(struct tipc_msg *m, u32 w,
                                u32 pos, u32 mask, u32 val)
{
        val = (val & mask) << pos;
        mask = mask << pos;
        m->hdr[w] &= ~htonl(mask);
        m->hdr[w] |= htonl(val);
}

/*
 * Word 0
 */
static inline u32 msg_version(struct tipc_msg *m)
{
        return msg_bits(m, 0, 29, 7);
}

static inline void msg_set_version(struct tipc_msg *m)
{
        msg_set_bits(m, 0, 29, 7, TIPC_VERSION);
}

static inline u32 msg_user(struct tipc_msg *m)
{
        return msg_bits(m, 0, 25, 0xf);
}

static inline u32 msg_isdata(struct tipc_msg *m)
{
        return msg_user(m) <= TIPC_CRITICAL_IMPORTANCE;
}

static inline void msg_set_user(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 0, 25, 0xf, n);
}

static inline u32 msg_hdr_sz(struct tipc_msg *m)
{
        return msg_bits(m, 0, 21, 0xf) << 2;
}

static inline void msg_set_hdr_sz(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 0, 21, 0xf, n>>2);
}

static inline u32 msg_size(struct tipc_msg *m)
{
        return msg_bits(m, 0, 0, 0x1ffff);
}

static inline u32 msg_blocks(struct tipc_msg *m)
{
        return (msg_size(m) / 1024) + 1;
}

static inline u32 msg_data_sz(struct tipc_msg *m)
{
        return msg_size(m) - msg_hdr_sz(m);
}

static inline int msg_non_seq(struct tipc_msg *m)
{
        return msg_bits(m, 0, 20, 1);
}

static inline void msg_set_non_seq(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 0, 20, 1, n);
}

static inline int msg_is_syn(struct tipc_msg *m)
{
        return msg_bits(m, 0, 17, 1);
}

static inline void msg_set_syn(struct tipc_msg *m, u32 d)
{
        msg_set_bits(m, 0, 17, 1, d);
}

static inline int msg_dest_droppable(struct tipc_msg *m)
{
        return msg_bits(m, 0, 19, 1);
}

static inline void msg_set_dest_droppable(struct tipc_msg *m, u32 d)
{
        msg_set_bits(m, 0, 19, 1, d);
}

static inline int msg_is_keepalive(struct tipc_msg *m)
{
        return msg_bits(m, 0, 19, 1);
}

static inline void msg_set_is_keepalive(struct tipc_msg *m, u32 d)
{
        msg_set_bits(m, 0, 19, 1, d);
}

static inline int msg_src_droppable(struct tipc_msg *m)
{
        return msg_bits(m, 0, 18, 1);
}

static inline void msg_set_src_droppable(struct tipc_msg *m, u32 d)
{
        msg_set_bits(m, 0, 18, 1, d);
}

static inline int msg_ack_required(struct tipc_msg *m)
{
        return msg_bits(m, 0, 18, 1);
}

static inline void msg_set_ack_required(struct tipc_msg *m)
{
        msg_set_bits(m, 0, 18, 1, 1);
}

static inline int msg_nagle_ack(struct tipc_msg *m)
{
        return msg_bits(m, 0, 18, 1);
}

static inline void msg_set_nagle_ack(struct tipc_msg *m)
{
        msg_set_bits(m, 0, 18, 1, 1);
}

static inline bool msg_is_rcast(struct tipc_msg *m)
{
        return msg_bits(m, 0, 18, 0x1);
}

static inline void msg_set_is_rcast(struct tipc_msg *m, bool d)
{
        msg_set_bits(m, 0, 18, 0x1, d);
}

static inline void msg_set_size(struct tipc_msg *m, u32 sz)
{
        m->hdr[0] = htonl((msg_word(m, 0) & ~0x1ffff) | sz);
}

static inline unchar *msg_data(struct tipc_msg *m)
{
        return ((unchar *)m) + msg_hdr_sz(m);
}

static inline struct tipc_msg *msg_inner_hdr(struct tipc_msg *m)
{
        return (struct tipc_msg *)msg_data(m);
}

/*
 * Word 1
 */
static inline u32 msg_type(struct tipc_msg *m)
{
        return msg_bits(m, 1, 29, 0x7);
}

static inline void msg_set_type(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 1, 29, 0x7, n);
}

static inline int msg_in_group(struct tipc_msg *m)
{
        int mtyp = msg_type(m);

        return mtyp >= TIPC_GRP_MEMBER_EVT && mtyp <= TIPC_GRP_UCAST_MSG;
}

static inline bool msg_is_grp_evt(struct tipc_msg *m)
{
        return msg_type(m) == TIPC_GRP_MEMBER_EVT;
}

static inline u32 msg_named(struct tipc_msg *m)
{
        return msg_type(m) == TIPC_NAMED_MSG;
}

static inline u32 msg_mcast(struct tipc_msg *m)
{
        int mtyp = msg_type(m);

        return ((mtyp == TIPC_MCAST_MSG) || (mtyp == TIPC_GRP_BCAST_MSG) ||
                (mtyp == TIPC_GRP_MCAST_MSG));
}

static inline u32 msg_connected(struct tipc_msg *m)
{
        return msg_type(m) == TIPC_CONN_MSG;
}

static inline u32 msg_direct(struct tipc_msg *m)
{
        return msg_type(m) == TIPC_DIRECT_MSG;
}

static inline u32 msg_errcode(struct tipc_msg *m)
{
        return msg_bits(m, 1, 25, 0xf);
}

static inline void msg_set_errcode(struct tipc_msg *m, u32 err)
{
        msg_set_bits(m, 1, 25, 0xf, err);
}

static inline void msg_set_bulk(struct tipc_msg *m)
{
        msg_set_bits(m, 1, 28, 0x1, 1);
}

static inline u32 msg_is_bulk(struct tipc_msg *m)
{
        return msg_bits(m, 1, 28, 0x1);
}

static inline void msg_set_last_bulk(struct tipc_msg *m)
{
        msg_set_bits(m, 1, 27, 0x1, 1);
}

static inline u32 msg_is_last_bulk(struct tipc_msg *m)
{
        return msg_bits(m, 1, 27, 0x1);
}

static inline void msg_set_non_legacy(struct tipc_msg *m)
{
        msg_set_bits(m, 1, 26, 0x1, 1);
}

static inline u32 msg_is_legacy(struct tipc_msg *m)
{
        return !msg_bits(m, 1, 26, 0x1);
}

static inline u32 msg_reroute_cnt(struct tipc_msg *m)
{
        return msg_bits(m, 1, 21, 0xf);
}

static inline void msg_incr_reroute_cnt(struct tipc_msg *m)
{
        msg_set_bits(m, 1, 21, 0xf, msg_reroute_cnt(m) + 1);
}

static inline u32 msg_lookup_scope(struct tipc_msg *m)
{
        return msg_bits(m, 1, 19, 0x3);
}

static inline void msg_set_lookup_scope(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 1, 19, 0x3, n);
}

static inline u16 msg_bcast_ack(struct tipc_msg *m)
{
        return msg_bits(m, 1, 0, 0xffff);
}

static inline void msg_set_bcast_ack(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 1, 0, 0xffff, n);
}

/* Note: reusing bits in word 1 for ACTIVATE_MSG only, to re-synch
 * link peer session number
 */
static inline bool msg_dest_session_valid(struct tipc_msg *m)
{
        return msg_bits(m, 1, 16, 0x1);
}

static inline void msg_set_dest_session_valid(struct tipc_msg *m, bool valid)
{
        msg_set_bits(m, 1, 16, 0x1, valid);
}

static inline u16 msg_dest_session(struct tipc_msg *m)
{
        return msg_bits(m, 1, 0, 0xffff);
}

static inline void msg_set_dest_session(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 1, 0, 0xffff, n);
}

/*
 * Word 2
 */
static inline u16 msg_ack(struct tipc_msg *m)
{
        return msg_bits(m, 2, 16, 0xffff);
}

static inline void msg_set_ack(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 2, 16, 0xffff, n);
}

static inline u16 msg_seqno(struct tipc_msg *m)
{
        return msg_bits(m, 2, 0, 0xffff);
}

static inline void msg_set_seqno(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 2, 0, 0xffff, n);
}

/*
 * Words 3-10
 */
static inline u32 msg_importance(struct tipc_msg *m)
{
        int usr = msg_user(m);

        if (likely((usr <= TIPC_CRITICAL_IMPORTANCE) && !msg_errcode(m)))
                return usr;
        if ((usr == MSG_FRAGMENTER) || (usr == MSG_BUNDLER))
                return msg_bits(m, 9, 0, 0x7);
        return TIPC_SYSTEM_IMPORTANCE;
}

static inline void msg_set_importance(struct tipc_msg *m, u32 i)
{
        int usr = msg_user(m);

        if (likely((usr == MSG_FRAGMENTER) || (usr == MSG_BUNDLER)))
                msg_set_bits(m, 9, 0, 0x7, i);
        else if (i < TIPC_SYSTEM_IMPORTANCE)
                msg_set_user(m, i);
        else
                pr_warn("Trying to set illegal importance in message\n");
}

static inline u32 msg_prevnode(struct tipc_msg *m)
{
        return msg_word(m, 3);
}

static inline void msg_set_prevnode(struct tipc_msg *m, u32 a)
{
        msg_set_word(m, 3, a);
}

static inline u32 msg_origport(struct tipc_msg *m)
{
        if (msg_user(m) == MSG_FRAGMENTER)
                m = msg_inner_hdr(m);
        return msg_word(m, 4);
}

static inline void msg_set_origport(struct tipc_msg *m, u32 p)
{
        msg_set_word(m, 4, p);
}

static inline u16 msg_named_seqno(struct tipc_msg *m)
{
        return msg_bits(m, 4, 0, 0xffff);
}

static inline void msg_set_named_seqno(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 4, 0, 0xffff, n);
}

static inline u32 msg_destport(struct tipc_msg *m)
{
        return msg_word(m, 5);
}

static inline void msg_set_destport(struct tipc_msg *m, u32 p)
{
        msg_set_word(m, 5, p);
}

static inline u32 msg_mc_netid(struct tipc_msg *m)
{
        return msg_word(m, 5);
}

static inline void msg_set_mc_netid(struct tipc_msg *m, u32 p)
{
        msg_set_word(m, 5, p);
}

static inline int msg_short(struct tipc_msg *m)
{
        return msg_hdr_sz(m) == SHORT_H_SIZE;
}

static inline u32 msg_orignode(struct tipc_msg *m)
{
        if (likely(msg_short(m)))
                return msg_prevnode(m);
        return msg_word(m, 6);
}

static inline void msg_set_orignode(struct tipc_msg *m, u32 a)
{
        msg_set_word(m, 6, a);
}

static inline u32 msg_destnode(struct tipc_msg *m)
{
        return msg_word(m, 7);
}

static inline void msg_set_destnode(struct tipc_msg *m, u32 a)
{
        msg_set_word(m, 7, a);
}

static inline u32 msg_nametype(struct tipc_msg *m)
{
        return msg_word(m, 8);
}

static inline void msg_set_nametype(struct tipc_msg *m, u32 n)
{
        msg_set_word(m, 8, n);
}

static inline u32 msg_nameinst(struct tipc_msg *m)
{
        return msg_word(m, 9);
}

static inline u32 msg_namelower(struct tipc_msg *m)
{
        return msg_nameinst(m);
}

static inline void msg_set_namelower(struct tipc_msg *m, u32 n)
{
        msg_set_word(m, 9, n);
}

static inline void msg_set_nameinst(struct tipc_msg *m, u32 n)
{
        msg_set_namelower(m, n);
}

static inline u32 msg_nameupper(struct tipc_msg *m)
{
        return msg_word(m, 10);
}

static inline void msg_set_nameupper(struct tipc_msg *m, u32 n)
{
        msg_set_word(m, 10, n);
}

/*
 * Constants and routines used to read and write TIPC internal message headers
 */

/*
 *  Connection management protocol message types
 */
#define CONN_PROBE        0
#define CONN_PROBE_REPLY  1
#define CONN_ACK          2

/*
 * Name distributor message types
 */
#define PUBLICATION       0
#define WITHDRAWAL        1

/*
 * Segmentation message types
 */
#define FIRST_FRAGMENT                0
#define FRAGMENT                1
#define LAST_FRAGMENT                2

/*
 * Link management protocol message types
 */
#define STATE_MSG                0
#define RESET_MSG                1
#define ACTIVATE_MSG                2

/*
 * Changeover tunnel message types
 */
#define SYNCH_MSG                0
#define FAILOVER_MSG                1

/*
 * Config protocol message types
 */
#define DSC_REQ_MSG                0
#define DSC_RESP_MSG                1
#define DSC_TRIAL_MSG                2
#define DSC_TRIAL_FAIL_MSG        3

/*
 * Group protocol message types
 */
#define GRP_JOIN_MSG         0
#define GRP_LEAVE_MSG        1
#define GRP_ADV_MSG          2
#define GRP_ACK_MSG          3
#define GRP_RECLAIM_MSG      4
#define GRP_REMIT_MSG        5

/* Crypto message types */
#define KEY_DISTR_MSG                0

/*
 * Word 1
 */
static inline u32 msg_seq_gap(struct tipc_msg *m)
{
        return msg_bits(m, 1, 16, 0x1fff);
}

static inline void msg_set_seq_gap(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 1, 16, 0x1fff, n);
}

static inline u32 msg_node_sig(struct tipc_msg *m)
{
        return msg_bits(m, 1, 0, 0xffff);
}

static inline void msg_set_node_sig(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 1, 0, 0xffff, n);
}

static inline u32 msg_node_capabilities(struct tipc_msg *m)
{
        return msg_bits(m, 1, 15, 0x1fff);
}

static inline void msg_set_node_capabilities(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 1, 15, 0x1fff, n);
}

/*
 * Word 2
 */
static inline u32 msg_dest_domain(struct tipc_msg *m)
{
        return msg_word(m, 2);
}

static inline void msg_set_dest_domain(struct tipc_msg *m, u32 n)
{
        msg_set_word(m, 2, n);
}

static inline void msg_set_bcgap_after(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 2, 16, 0xffff, n);
}

static inline u32 msg_bcgap_to(struct tipc_msg *m)
{
        return msg_bits(m, 2, 0, 0xffff);
}

static inline void msg_set_bcgap_to(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 2, 0, 0xffff, n);
}

/*
 * Word 4
 */
static inline u32 msg_last_bcast(struct tipc_msg *m)
{
        return msg_bits(m, 4, 16, 0xffff);
}

static inline u32 msg_bc_snd_nxt(struct tipc_msg *m)
{
        return msg_last_bcast(m) + 1;
}

static inline void msg_set_last_bcast(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 4, 16, 0xffff, n);
}

static inline u32 msg_nof_fragms(struct tipc_msg *m)
{
        return msg_bits(m, 4, 0, 0xffff);
}

static inline void msg_set_nof_fragms(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 4, 0, 0xffff, n);
}

static inline u32 msg_fragm_no(struct tipc_msg *m)
{
        return msg_bits(m, 4, 16, 0xffff);
}

static inline void msg_set_fragm_no(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 4, 16, 0xffff, n);
}

static inline u16 msg_next_sent(struct tipc_msg *m)
{
        return msg_bits(m, 4, 0, 0xffff);
}

static inline void msg_set_next_sent(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 4, 0, 0xffff, n);
}

static inline u32 msg_bc_netid(struct tipc_msg *m)
{
        return msg_word(m, 4);
}

static inline void msg_set_bc_netid(struct tipc_msg *m, u32 id)
{
        msg_set_word(m, 4, id);
}

static inline u32 msg_link_selector(struct tipc_msg *m)
{
        if (msg_user(m) == MSG_FRAGMENTER)
                m = (void *)msg_data(m);
        return msg_bits(m, 4, 0, 1);
}

/*
 * Word 5
 */
static inline u16 msg_session(struct tipc_msg *m)
{
        return msg_bits(m, 5, 16, 0xffff);
}

static inline void msg_set_session(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 5, 16, 0xffff, n);
}

static inline u32 msg_probe(struct tipc_msg *m)
{
        return msg_bits(m, 5, 0, 1);
}

static inline void msg_set_probe(struct tipc_msg *m, u32 val)
{
        msg_set_bits(m, 5, 0, 1, val);
}

static inline char msg_net_plane(struct tipc_msg *m)
{
        return msg_bits(m, 5, 1, 7) + 'A';
}

static inline void msg_set_net_plane(struct tipc_msg *m, char n)
{
        msg_set_bits(m, 5, 1, 7, (n - 'A'));
}

static inline u32 msg_linkprio(struct tipc_msg *m)
{
        return msg_bits(m, 5, 4, 0x1f);
}

static inline void msg_set_linkprio(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 5, 4, 0x1f, n);
}

static inline u32 msg_bearer_id(struct tipc_msg *m)
{
        return msg_bits(m, 5, 9, 0x7);
}

static inline void msg_set_bearer_id(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 5, 9, 0x7, n);
}

static inline u32 msg_redundant_link(struct tipc_msg *m)
{
        return msg_bits(m, 5, 12, 0x1);
}

static inline void msg_set_redundant_link(struct tipc_msg *m, u32 r)
{
        msg_set_bits(m, 5, 12, 0x1, r);
}

static inline u32 msg_peer_stopping(struct tipc_msg *m)
{
        return msg_bits(m, 5, 13, 0x1);
}

static inline void msg_set_peer_stopping(struct tipc_msg *m, u32 s)
{
        msg_set_bits(m, 5, 13, 0x1, s);
}

static inline bool msg_bc_ack_invalid(struct tipc_msg *m)
{
        switch (msg_user(m)) {
        case BCAST_PROTOCOL:
        case NAME_DISTRIBUTOR:
        case LINK_PROTOCOL:
                return msg_bits(m, 5, 14, 0x1);
        default:
                return false;
        }
}

static inline void msg_set_bc_ack_invalid(struct tipc_msg *m, bool invalid)
{
        msg_set_bits(m, 5, 14, 0x1, invalid);
}

static inline char *msg_media_addr(struct tipc_msg *m)
{
        return (char *)&m->hdr[TIPC_MEDIA_INFO_OFFSET];
}

static inline u32 msg_bc_gap(struct tipc_msg *m)
{
        return msg_bits(m, 8, 0, 0x3ff);
}

static inline void msg_set_bc_gap(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 8, 0, 0x3ff, n);
}

/*
 * Word 9
 */
static inline u16 msg_msgcnt(struct tipc_msg *m)
{
        return msg_bits(m, 9, 16, 0xffff);
}

static inline void msg_set_msgcnt(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 9, 16, 0xffff, n);
}

static inline u16 msg_syncpt(struct tipc_msg *m)
{
        return msg_bits(m, 9, 16, 0xffff);
}

static inline void msg_set_syncpt(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 9, 16, 0xffff, n);
}

static inline u32 msg_conn_ack(struct tipc_msg *m)
{
        return msg_bits(m, 9, 16, 0xffff);
}

static inline void msg_set_conn_ack(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 9, 16, 0xffff, n);
}

static inline u16 msg_adv_win(struct tipc_msg *m)
{
        return msg_bits(m, 9, 0, 0xffff);
}

static inline void msg_set_adv_win(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 9, 0, 0xffff, n);
}

static inline u32 msg_max_pkt(struct tipc_msg *m)
{
        return msg_bits(m, 9, 16, 0xffff) * 4;
}

static inline void msg_set_max_pkt(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 9, 16, 0xffff, (n / 4));
}

static inline u32 msg_link_tolerance(struct tipc_msg *m)
{
        return msg_bits(m, 9, 0, 0xffff);
}

static inline void msg_set_link_tolerance(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 9, 0, 0xffff, n);
}

static inline u16 msg_grp_bc_syncpt(struct tipc_msg *m)
{
        return msg_bits(m, 9, 16, 0xffff);
}

static inline void msg_set_grp_bc_syncpt(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 9, 16, 0xffff, n);
}

static inline u16 msg_grp_bc_acked(struct tipc_msg *m)
{
        return msg_bits(m, 9, 16, 0xffff);
}

static inline void msg_set_grp_bc_acked(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 9, 16, 0xffff, n);
}

static inline u16 msg_grp_remitted(struct tipc_msg *m)
{
        return msg_bits(m, 9, 16, 0xffff);
}

static inline void msg_set_grp_remitted(struct tipc_msg *m, u16 n)
{
        msg_set_bits(m, 9, 16, 0xffff, n);
}

/* Word 10
 */
static inline u16 msg_grp_evt(struct tipc_msg *m)
{
        return msg_bits(m, 10, 0, 0x3);
}

static inline void msg_set_grp_evt(struct tipc_msg *m, int n)
{
        msg_set_bits(m, 10, 0, 0x3, n);
}

static inline u16 msg_grp_bc_ack_req(struct tipc_msg *m)
{
        return msg_bits(m, 10, 0, 0x1);
}

static inline void msg_set_grp_bc_ack_req(struct tipc_msg *m, bool n)
{
        msg_set_bits(m, 10, 0, 0x1, n);
}

static inline u16 msg_grp_bc_seqno(struct tipc_msg *m)
{
        return msg_bits(m, 10, 16, 0xffff);
}

static inline void msg_set_grp_bc_seqno(struct tipc_msg *m, u32 n)
{
        msg_set_bits(m, 10, 16, 0xffff, n);
}

static inline bool msg_peer_link_is_up(struct tipc_msg *m)
{
        if (likely(msg_user(m) != LINK_PROTOCOL))
                return true;
        if (msg_type(m) == STATE_MSG)
                return true;
        return false;
}

static inline bool msg_peer_node_is_up(struct tipc_msg *m)
{
        if (msg_peer_link_is_up(m))
                return true;
        return msg_redundant_link(m);
}

static inline bool msg_is_reset(struct tipc_msg *hdr)
{
        return (msg_user(hdr) == LINK_PROTOCOL) && (msg_type(hdr) == RESET_MSG);
}

/* Word 13
 */
static inline void msg_set_peer_net_hash(struct tipc_msg *m, u32 n)
{
        msg_set_word(m, 13, n);
}

static inline u32 msg_peer_net_hash(struct tipc_msg *m)
{
        return msg_word(m, 13);
}

/* Word 14
 */
static inline u32 msg_sugg_node_addr(struct tipc_msg *m)
{
        return msg_word(m, 14);
}

static inline void msg_set_sugg_node_addr(struct tipc_msg *m, u32 n)
{
        msg_set_word(m, 14, n);
}

static inline void msg_set_node_id(struct tipc_msg *hdr, u8 *id)
{
        memcpy(msg_data(hdr), id, 16);
}

static inline u8 *msg_node_id(struct tipc_msg *hdr)
{
        return (u8 *)msg_data(hdr);
}

struct sk_buff *tipc_buf_acquire(u32 size, gfp_t gfp);
bool tipc_msg_validate(struct sk_buff **_skb);
bool tipc_msg_reverse(u32 own_addr, struct sk_buff **skb, int err);
void tipc_skb_reject(struct net *net, int err, struct sk_buff *skb,
                     struct sk_buff_head *xmitq);
void tipc_msg_init(u32 own_addr, struct tipc_msg *m, u32 user, u32 type,
                   u32 hsize, u32 destnode);
struct sk_buff *tipc_msg_create(uint user, uint type, uint hdr_sz,
                                uint data_sz, u32 dnode, u32 onode,
                                u32 dport, u32 oport, int errcode);
int tipc_buf_append(struct sk_buff **headbuf, struct sk_buff **buf);
bool tipc_msg_try_bundle(struct sk_buff *tskb, struct sk_buff **skb, u32 mss,
                         u32 dnode, bool *new_bundle);
bool tipc_msg_extract(struct sk_buff *skb, struct sk_buff **iskb, int *pos);
int tipc_msg_fragment(struct sk_buff *skb, const struct tipc_msg *hdr,
                      int pktmax, struct sk_buff_head *frags);
int tipc_msg_build(struct tipc_msg *mhdr, struct msghdr *m,
                   int offset, int dsz, int mtu, struct sk_buff_head *list);
int tipc_msg_append(struct tipc_msg *hdr, struct msghdr *m, int dlen,
                    int mss, struct sk_buff_head *txq);
bool tipc_msg_lookup_dest(struct net *net, struct sk_buff *skb, int *err);
bool tipc_msg_assemble(struct sk_buff_head *list);
bool tipc_msg_reassemble(struct sk_buff_head *list, struct sk_buff_head *rcvq);
bool tipc_msg_pskb_copy(u32 dst, struct sk_buff_head *msg,
                        struct sk_buff_head *cpy);
bool __tipc_skb_queue_sorted(struct sk_buff_head *list, u16 seqno,
                             struct sk_buff *skb);
bool tipc_msg_skb_clone(struct sk_buff_head *msg, struct sk_buff_head *cpy);

static inline u16 buf_seqno(struct sk_buff *skb)
{
        return msg_seqno(buf_msg(skb));
}

static inline int buf_roundup_len(struct sk_buff *skb)
{
        return (skb->len / 1024 + 1) * 1024;
}

/* tipc_skb_peek(): peek and reserve first buffer in list
 * @list: list to be peeked in
 * Returns pointer to first buffer in list, if any
 */
static inline struct sk_buff *tipc_skb_peek(struct sk_buff_head *list,
                                            spinlock_t *lock)
{
        struct sk_buff *skb;

        spin_lock_bh(lock);
        skb = skb_peek(list);
        if (skb)
                skb_get(skb);
        spin_unlock_bh(lock);
        return skb;
}

/* tipc_skb_peek_port(): find a destination port, ignoring all destinations
 *                       up to and including 'filter'.
 * Note: ignoring previously tried destinations minimizes the risk of
 *       contention on the socket lock
 * @list: list to be peeked in
 * @filter: last destination to be ignored from search
 * Returns a destination port number, of applicable.
 */
static inline u32 tipc_skb_peek_port(struct sk_buff_head *list, u32 filter)
{
        struct sk_buff *skb;
        u32 dport = 0;
        bool ignore = true;

        spin_lock_bh(&list->lock);
        skb_queue_walk(list, skb) {
                dport = msg_destport(buf_msg(skb));
                if (!filter || skb_queue_is_last(list, skb))
                        break;
                if (dport == filter)
                        ignore = false;
                else if (!ignore)
                        break;
        }
        spin_unlock_bh(&list->lock);
        return dport;
}

/* tipc_skb_dequeue(): unlink first buffer with dest 'dport' from list
 * @list: list to be unlinked from
 * @dport: selection criteria for buffer to unlink
 */
static inline struct sk_buff *tipc_skb_dequeue(struct sk_buff_head *list,
                                               u32 dport)
{
        struct sk_buff *_skb, *tmp, *skb = NULL;

        spin_lock_bh(&list->lock);
        skb_queue_walk_safe(list, _skb, tmp) {
                if (msg_destport(buf_msg(_skb)) == dport) {
                        __skb_unlink(_skb, list);
                        skb = _skb;
                        break;
                }
        }
        spin_unlock_bh(&list->lock);
        return skb;
}

/* tipc_skb_queue_splice_tail - append an skb list to lock protected list
 * @list: the new list to append. Not lock protected
 * @head: target list. Lock protected.
 */
static inline void tipc_skb_queue_splice_tail(struct sk_buff_head *list,
                                              struct sk_buff_head *head)
{
        spin_lock_bh(&head->lock);
        skb_queue_splice_tail(list, head);
        spin_unlock_bh(&head->lock);
}

/* tipc_skb_queue_splice_tail_init - merge two lock protected skb lists
 * @list: the new list to add. Lock protected. Will be reinitialized
 * @head: target list. Lock protected.
 */
static inline void tipc_skb_queue_splice_tail_init(struct sk_buff_head *list,
                                                   struct sk_buff_head *head)
{
        struct sk_buff_head tmp;

        __skb_queue_head_init(&tmp);

        spin_lock_bh(&list->lock);
        skb_queue_splice_tail_init(list, &tmp);
        spin_unlock_bh(&list->lock);
        tipc_skb_queue_splice_tail(&tmp, head);
}

/* __tipc_skb_dequeue() - dequeue the head skb according to expected seqno
 * @list: list to be dequeued from
 * @seqno: seqno of the expected msg
 *
 * returns skb dequeued from the list if its seqno is less than or equal to
 * the expected one, otherwise the skb is still hold
 *
 * Note: must be used with appropriate locks held only
 */
static inline struct sk_buff *__tipc_skb_dequeue(struct sk_buff_head *list,
                                                 u16 seqno)
{
        struct sk_buff *skb = skb_peek(list);

        if (skb && less_eq(buf_seqno(skb), seqno)) {
                __skb_unlink(skb, list);
                return skb;
        }
        return NULL;
}

#endif























































































































   96 








   96 


   96 





























   96 

   96 




























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
// SPDX-License-Identifier: GPL-2.0
/*
 * linux/drivers/char/misc.c
 *
 * Generic misc open routine by Johan Myreen
 *
 * Based on code from Linus
 *
 * Teemu Rantanen's Microsoft Busmouse support and Derrick Cole's
 *   changes incorporated into 0.97pl4
 *   by Peter Cervasio (pete%q106fm.uucp@wupost.wustl.edu) (08SEP92)
 *   See busmouse.c for particulars.
 *
 * Made things a lot mode modular - easy to compile in just one or two
 * of the misc drivers, as they are now completely independent. Linus.
 *
 * Support for loadable modules. 8-Sep-95 Philip Blundell <pjb27@cam.ac.uk>
 *
 * Fixed a failing symbol register to free the device registration
 *                Alan Cox <alan@lxorguk.ukuu.org.uk> 21-Jan-96
 *
 * Dynamic minors and /proc/mice by Alessandro Rubini. 26-Mar-96
 *
 * Renamed to misc and miscdevice to be more accurate. Alan Cox 26-Mar-96
 *
 * Handling of mouse minor numbers for kerneld:
 *  Idea by Jacques Gelinas <jack@solucorp.qc.ca>,
 *  adapted by Bjorn Ekwall <bj0rn@blox.se>
 *  corrected by Alan Cox <alan@lxorguk.ukuu.org.uk>
 *
 * Changes for kmod (from kerneld):
 *        Cyrus Durgin <cider@speakeasy.org>
 *
 * Added devfs support. Richard Gooch <rgooch@atnf.csiro.au>  10-Jan-1998
 */

#include <linux/module.h>

#include <linux/fs.h>
#include <linux/errno.h>
#include <linux/miscdevice.h>
#include <linux/kernel.h>
#include <linux/major.h>
#include <linux/mutex.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/stat.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/tty.h>
#include <linux/kmod.h>
#include <linux/gfp.h>

/*
 * Head entry for the doubly linked miscdevice list
 */
static LIST_HEAD(misc_list);
static DEFINE_MUTEX(misc_mtx);

/*
 * Assigned numbers.
 */
static DEFINE_IDA(misc_minors_ida);

static int misc_minor_alloc(int minor)
{
        int ret = 0;

        if (minor == MISC_DYNAMIC_MINOR) {
                /* allocate free id */
                ret = ida_alloc_range(&misc_minors_ida, MISC_DYNAMIC_MINOR + 1,
                                      MINORMASK, GFP_KERNEL);
        } else {
                ret = ida_alloc_range(&misc_minors_ida, minor, minor, GFP_KERNEL);
        }
        return ret;
}

static void misc_minor_free(int minor)
{
        ida_free(&misc_minors_ida, minor);
}

#ifdef CONFIG_PROC_FS
static void *misc_seq_start(struct seq_file *seq, loff_t *pos)
{
        mutex_lock(&misc_mtx);
        return seq_list_start(&misc_list, *pos);
}

static void *misc_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        return seq_list_next(v, &misc_list, pos);
}

static void misc_seq_stop(struct seq_file *seq, void *v)
{
        mutex_unlock(&misc_mtx);
}

static int misc_seq_show(struct seq_file *seq, void *v)
{
        const struct miscdevice *p = list_entry(v, struct miscdevice, list);

        seq_printf(seq, "%3i %s\n", p->minor, p->name ? p->name : "");
        return 0;
}


static const struct seq_operations misc_seq_ops = {
        .start = misc_seq_start,
        .next  = misc_seq_next,
        .stop  = misc_seq_stop,
        .show  = misc_seq_show,
};
#endif

static int misc_open(struct inode *inode, struct file *file)
{
        int minor = iminor(inode);
        struct miscdevice *c = NULL, *iter;
        int err = -ENODEV;
        const struct file_operations *new_fops = NULL;

        mutex_lock(&misc_mtx);

        list_for_each_entry(iter, &misc_list, list) {
                if (iter->minor != minor)
                        continue;
                c = iter;
                new_fops = fops_get(iter->fops);
                break;
        }

        /* Only request module for fixed minor code */
        if (!new_fops && minor < MISC_DYNAMIC_MINOR) {
                mutex_unlock(&misc_mtx);
                request_module("char-major-%d-%d", MISC_MAJOR, minor);
                mutex_lock(&misc_mtx);

                list_for_each_entry(iter, &misc_list, list) {
                        if (iter->minor != minor)
                                continue;
                        c = iter;
                        new_fops = fops_get(iter->fops);
                        break;
                }
        }

        if (!new_fops)
                goto fail;

        /*
         * Place the miscdevice in the file's
         * private_data so it can be used by the
         * file operations, including f_op->open below
         */
        file->private_data = c;

        err = 0;
        replace_fops(file, new_fops);
        if (file->f_op->open)
                err = file->f_op->open(inode, file);
fail:
        mutex_unlock(&misc_mtx);
        return err;
}

static char *misc_devnode(const struct device *dev, umode_t *mode)
{
        const struct miscdevice *c = dev_get_drvdata(dev);

        if (mode && c->mode)
                *mode = c->mode;
        if (c->nodename)
                return kstrdup(c->nodename, GFP_KERNEL);
        return NULL;
}

static const struct class misc_class = {
        .name                = "misc",
        .devnode        = misc_devnode,
};

static const struct file_operations misc_fops = {
        .owner                = THIS_MODULE,
        .open                = misc_open,
        .llseek                = noop_llseek,
};

/**
 *        misc_register        -        register a miscellaneous device
 *        @misc: device structure
 *
 *        Register a miscellaneous device with the kernel. If the minor
 *        number is set to %MISC_DYNAMIC_MINOR a minor number is assigned
 *        and placed in the minor field of the structure. For other cases
 *        the minor number requested is used.
 *
 *        The structure passed is linked into the kernel and may not be
 *        destroyed until it has been unregistered. By default, an open()
 *        syscall to the device sets file->private_data to point to the
 *        structure. Drivers don't need open in fops for this.
 *
 *        A zero is returned on success and a negative errno code for
 *        failure.
 */

int misc_register(struct miscdevice *misc)
{
        dev_t dev;
        int err = 0;
        bool is_dynamic = (misc->minor == MISC_DYNAMIC_MINOR);

        if (misc->minor > MISC_DYNAMIC_MINOR) {
                pr_err("Invalid fixed minor %d for miscdevice '%s'\n",
                       misc->minor, misc->name);
                return -EINVAL;
        }

        INIT_LIST_HEAD(&misc->list);

        mutex_lock(&misc_mtx);

        if (is_dynamic) {
                int i = misc_minor_alloc(misc->minor);

                if (i < 0) {
                        err = -EBUSY;
                        goto out;
                }
                misc->minor = i;
        } else {
                struct miscdevice *c;
                int i;

                list_for_each_entry(c, &misc_list, list) {
                        if (c->minor == misc->minor) {
                                err = -EBUSY;
                                goto out;
                        }
                }

                i = misc_minor_alloc(misc->minor);
                if (i < 0) {
                        err = -EBUSY;
                        goto out;
                }
        }

        dev = MKDEV(MISC_MAJOR, misc->minor);

        misc->this_device =
                device_create_with_groups(&misc_class, misc->parent, dev,
                                          misc, misc->groups, "%s", misc->name);
        if (IS_ERR(misc->this_device)) {
                misc_minor_free(misc->minor);
                if (is_dynamic) {
                        misc->minor = MISC_DYNAMIC_MINOR;
                }
                err = PTR_ERR(misc->this_device);
                goto out;
        }

        /*
         * Add it to the front, so that later devices can "override"
         * earlier defaults
         */
        list_add(&misc->list, &misc_list);
 out:
        mutex_unlock(&misc_mtx);
        return err;
}
EXPORT_SYMBOL(misc_register);

/**
 *        misc_deregister - unregister a miscellaneous device
 *        @misc: device to unregister
 *
 *        Unregister a miscellaneous device that was previously
 *        successfully registered with misc_register().
 */

void misc_deregister(struct miscdevice *misc)
{
        mutex_lock(&misc_mtx);
        list_del_init(&misc->list);
        device_destroy(&misc_class, MKDEV(MISC_MAJOR, misc->minor));
        misc_minor_free(misc->minor);
        if (misc->minor > MISC_DYNAMIC_MINOR)
                misc->minor = MISC_DYNAMIC_MINOR;
        mutex_unlock(&misc_mtx);
}
EXPORT_SYMBOL(misc_deregister);

static int __init misc_init(void)
{
        int err;
        struct proc_dir_entry *misc_proc_file;

        misc_proc_file = proc_create_seq("misc", 0, NULL, &misc_seq_ops);
        err = class_register(&misc_class);
        if (err)
                goto fail_remove;

        err = __register_chrdev(MISC_MAJOR, 0, MINORMASK + 1, "misc", &misc_fops);
        if (err < 0)
                goto fail_printk;
        return 0;

fail_printk:
        pr_err("unable to get major %d for misc devices\n", MISC_MAJOR);
        class_unregister(&misc_class);
fail_remove:
        if (misc_proc_file)
                remove_proc_entry("misc", NULL);
        return err;
}
subsys_initcall(misc_init);
































  161 




   78 




   84 







































  585 


  583 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Lock-less NULL terminated single linked list
 *
 * The basic atomic operation of this list is cmpxchg on long.  On
 * architectures that don't have NMI-safe cmpxchg implementation, the
 * list can NOT be used in NMI handlers.  So code that uses the list in
 * an NMI handler should depend on CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG.
 *
 * Copyright 2010,2011 Intel Corp.
 *   Author: Huang Ying <ying.huang@intel.com>
 */
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/llist.h>

/**
 * llist_del_first - delete the first entry of lock-less list
 * @head:        the head for your lock-less list
 *
 * If list is empty, return NULL, otherwise, return the first entry
 * deleted, this is the newest added one.
 *
 * Only one llist_del_first user can be used simultaneously with
 * multiple llist_add users without lock.  Because otherwise
 * llist_del_first, llist_add, llist_add (or llist_del_all, llist_add,
 * llist_add) sequence in another user may change @head->first->next,
 * but keep @head->first.  If multiple consumers are needed, please
 * use llist_del_all or use lock between consumers.
 */
struct llist_node *llist_del_first(struct llist_head *head)
{
        struct llist_node *entry, *next;

        entry = smp_load_acquire(&head->first);
        do {
                if (entry == NULL)
                        return NULL;
                next = READ_ONCE(entry->next);
        } while (!try_cmpxchg(&head->first, &entry, next));

        return entry;
}
EXPORT_SYMBOL_GPL(llist_del_first);

/**
 * llist_del_first_this - delete given entry of lock-less list if it is first
 * @head:        the head for your lock-less list
 * @this:        a list entry.
 *
 * If head of the list is given entry, delete and return %true else
 * return %false.
 *
 * Multiple callers can safely call this concurrently with multiple
 * llist_add() callers, providing all the callers offer a different @this.
 */
bool llist_del_first_this(struct llist_head *head,
                          struct llist_node *this)
{
        struct llist_node *entry, *next;

        /* acquire ensures orderig wrt try_cmpxchg() is llist_del_first() */
        entry = smp_load_acquire(&head->first);
        do {
                if (entry != this)
                        return false;
                next = READ_ONCE(entry->next);
        } while (!try_cmpxchg(&head->first, &entry, next));

        return true;
}
EXPORT_SYMBOL_GPL(llist_del_first_this);

/**
 * llist_reverse_order - reverse order of a llist chain
 * @head:        first item of the list to be reversed
 *
 * Reverse the order of a chain of llist entries and return the
 * new first entry.
 */
struct llist_node *llist_reverse_order(struct llist_node *head)
{
        struct llist_node *new_head = NULL;

        while (head) {
                struct llist_node *tmp = head;
                head = head->next;
                tmp->next = new_head;
                new_head = tmp;
        }

        return new_head;
}
EXPORT_SYMBOL_GPL(llist_reverse_order);











































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (c) 2007-2017 Nicira, Inc.
 */

#ifndef FLOW_H
#define FLOW_H 1

#include <linux/cache.h>
#include <linux/kernel.h>
#include <linux/netlink.h>
#include <linux/openvswitch.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/rcupdate.h>
#include <linux/if_ether.h>
#include <linux/in6.h>
#include <linux/jiffies.h>
#include <linux/time.h>
#include <linux/cpumask.h>
#include <net/inet_ecn.h>
#include <net/ip_tunnels.h>
#include <net/dst_metadata.h>
#include <net/nsh.h>

struct sk_buff;

enum sw_flow_mac_proto {
        MAC_PROTO_NONE = 0,
        MAC_PROTO_ETHERNET,
};
#define SW_FLOW_KEY_INVALID        0x80
#define MPLS_LABEL_DEPTH       3

/* Bit definitions for IPv6 Extension Header pseudo-field. */
enum ofp12_ipv6exthdr_flags {
        OFPIEH12_NONEXT = 1 << 0,   /* "No next header" encountered. */
        OFPIEH12_ESP    = 1 << 1,   /* Encrypted Sec Payload header present. */
        OFPIEH12_AUTH   = 1 << 2,   /* Authentication header present. */
        OFPIEH12_DEST   = 1 << 3,   /* 1 or 2 dest headers present. */
        OFPIEH12_FRAG   = 1 << 4,   /* Fragment header present. */
        OFPIEH12_ROUTER = 1 << 5,   /* Router header present. */
        OFPIEH12_HOP    = 1 << 6,   /* Hop-by-hop header present. */
        OFPIEH12_UNREP  = 1 << 7,   /* Unexpected repeats encountered. */
        OFPIEH12_UNSEQ  = 1 << 8    /* Unexpected sequencing encountered. */
};

/* Store options at the end of the array if they are less than the
 * maximum size. This allows us to get the benefits of variable length
 * matching for small options.
 */
#define TUN_METADATA_OFFSET(opt_len) \
        (sizeof_field(struct sw_flow_key, tun_opts) - opt_len)
#define TUN_METADATA_OPTS(flow_key, opt_len) \
        ((void *)((flow_key)->tun_opts + TUN_METADATA_OFFSET(opt_len)))

struct ovs_tunnel_info {
        struct metadata_dst        *tun_dst;
};

struct vlan_head {
        __be16 tpid; /* Vlan type. Generally 802.1q or 802.1ad.*/
        __be16 tci;  /* 0 if no VLAN, VLAN_CFI_MASK set otherwise. */
};

#define OVS_SW_FLOW_KEY_METADATA_SIZE                        \
        (offsetof(struct sw_flow_key, recirc_id) +        \
        sizeof_field(struct sw_flow_key, recirc_id))

struct ovs_key_nsh {
        struct ovs_nsh_key_base base;
        __be32 context[NSH_MD1_CONTEXT_SIZE];
};

struct sw_flow_key {
        u8 tun_opts[IP_TUNNEL_OPTS_MAX];
        u8 tun_opts_len;
        struct ip_tunnel_key tun_key;        /* Encapsulating tunnel key. */
        struct {
                u32        priority;        /* Packet QoS priority. */
                u32        skb_mark;        /* SKB mark. */
                u16        in_port;        /* Input switch port (or DP_MAX_PORTS). */
        } __packed phy; /* Safe when right after 'tun_key'. */
        u8 mac_proto;                        /* MAC layer protocol (e.g. Ethernet). */
        u8 tun_proto;                        /* Protocol of encapsulating tunnel. */
        u32 ovs_flow_hash;                /* Datapath computed hash value.  */
        u32 recirc_id;                        /* Recirculation ID.  */
        struct {
                u8     src[ETH_ALEN];        /* Ethernet source address. */
                u8     dst[ETH_ALEN];        /* Ethernet destination address. */
                struct vlan_head vlan;
                struct vlan_head cvlan;
                __be16 type;                /* Ethernet frame type. */
        } eth;
        /* Filling a hole of two bytes. */
        u8 ct_state;
        u8 ct_orig_proto;                /* CT original direction tuple IP
                                         * protocol.
                                         */
        union {
                struct {
                        u8     proto;        /* IP protocol or lower 8 bits of ARP opcode. */
                        u8     tos;            /* IP ToS. */
                        u8     ttl;            /* IP TTL/hop limit. */
                        u8     frag;        /* One of OVS_FRAG_TYPE_*. */
                } ip;
        };
        u16 ct_zone;                        /* Conntrack zone. */
        struct {
                __be16 src;                /* TCP/UDP/SCTP source port. */
                __be16 dst;                /* TCP/UDP/SCTP destination port. */
                __be16 flags;                /* TCP flags. */
        } tp;
        union {
                struct {
                        struct {
                                __be32 src;        /* IP source address. */
                                __be32 dst;        /* IP destination address. */
                        } addr;
                        union {
                                struct {
                                        __be32 src;
                                        __be32 dst;
                                } ct_orig;        /* Conntrack original direction fields. */
                                struct {
                                        u8 sha[ETH_ALEN];        /* ARP source hardware address. */
                                        u8 tha[ETH_ALEN];        /* ARP target hardware address. */
                                } arp;
                        };
                } ipv4;
                struct {
                        struct {
                                struct in6_addr src;        /* IPv6 source address. */
                                struct in6_addr dst;        /* IPv6 destination address. */
                        } addr;
                        __be32 label;                        /* IPv6 flow label. */
                        u16 exthdrs;        /* IPv6 extension header flags */
                        union {
                                struct {
                                        struct in6_addr src;
                                        struct in6_addr dst;
                                } ct_orig;        /* Conntrack original direction fields. */
                                struct {
                                        struct in6_addr target;        /* ND target address. */
                                        u8 sll[ETH_ALEN];        /* ND source link layer address. */
                                        u8 tll[ETH_ALEN];        /* ND target link layer address. */
                                } nd;
                        };
                } ipv6;
                struct {
                        u32 num_labels_mask;    /* labels present bitmap of effective length MPLS_LABEL_DEPTH */
                        __be32 lse[MPLS_LABEL_DEPTH];     /* label stack entry  */
                } mpls;

                struct ovs_key_nsh nsh;         /* network service header */
        };
        struct {
                /* Connection tracking fields not packed above. */
                struct {
                        __be16 src;        /* CT orig tuple tp src port. */
                        __be16 dst;        /* CT orig tuple tp dst port. */
                } orig_tp;
                u32 mark;
                struct ovs_key_ct_labels labels;
        } ct;

} __aligned(BITS_PER_LONG/8); /* Ensure that we can do comparisons as longs. */

static inline bool sw_flow_key_is_nd(const struct sw_flow_key *key)
{
        return key->eth.type == htons(ETH_P_IPV6) &&
                key->ip.proto == NEXTHDR_ICMP &&
                key->tp.dst == 0 &&
                (key->tp.src == htons(NDISC_NEIGHBOUR_SOLICITATION) ||
                 key->tp.src == htons(NDISC_NEIGHBOUR_ADVERTISEMENT));
}

struct sw_flow_key_range {
        unsigned short int start;
        unsigned short int end;
};

struct sw_flow_mask {
        int ref_count;
        struct rcu_head rcu;
        struct sw_flow_key_range range;
        struct sw_flow_key key;
};

struct sw_flow_match {
        struct sw_flow_key *key;
        struct sw_flow_key_range range;
        struct sw_flow_mask *mask;
};

#define MAX_UFID_LENGTH 16 /* 128 bits */

struct sw_flow_id {
        u32 ufid_len;
        union {
                u32 ufid[MAX_UFID_LENGTH / 4];
                struct sw_flow_key *unmasked_key;
        };
};

struct sw_flow_actions {
        struct rcu_head rcu;
        size_t orig_len;        /* From flow_cmd_new netlink actions size */
        u32 actions_len;
        struct nlattr actions[];
};

struct sw_flow_stats {
        u64 packet_count;                /* Number of packets matched. */
        u64 byte_count;                        /* Number of bytes matched. */
        unsigned long used;                /* Last used time (in jiffies). */
        spinlock_t lock;                /* Lock for atomic stats update. */
        __be16 tcp_flags;                /* Union of seen TCP flags. */
};

struct sw_flow {
        struct rcu_head rcu;
        struct {
                struct hlist_node node[2];
                u32 hash;
        } flow_table, ufid_table;
        int stats_last_writer;                /* CPU id of the last writer on
                                         * 'stats[0]'.
                                         */
        struct sw_flow_key key;
        struct sw_flow_id id;
        struct cpumask *cpu_used_mask;
        struct sw_flow_mask *mask;
        struct sw_flow_actions __rcu *sf_acts;
        struct sw_flow_stats __rcu *stats[]; /* One for each CPU.  First one
                                           * is allocated at flow creation time,
                                           * the rest are allocated on demand
                                           * while holding the 'stats[0].lock'.
                                           */
};

struct arp_eth_header {
        __be16      ar_hrd;        /* format of hardware address   */
        __be16      ar_pro;        /* format of protocol address   */
        unsigned char   ar_hln;        /* length of hardware address   */
        unsigned char   ar_pln;        /* length of protocol address   */
        __be16      ar_op;        /* ARP opcode (command)     */

        /* Ethernet+IPv4 specific members. */
        unsigned char       ar_sha[ETH_ALEN];        /* sender hardware address  */
        unsigned char       ar_sip[4];                /* sender IP address        */
        unsigned char       ar_tha[ETH_ALEN];        /* target hardware address  */
        unsigned char       ar_tip[4];                /* target IP address        */
} __packed;

static inline u8 ovs_key_mac_proto(const struct sw_flow_key *key)
{
        return key->mac_proto & ~SW_FLOW_KEY_INVALID;
}

static inline u16 __ovs_mac_header_len(u8 mac_proto)
{
        return mac_proto == MAC_PROTO_ETHERNET ? ETH_HLEN : 0;
}

static inline u16 ovs_mac_header_len(const struct sw_flow_key *key)
{
        return __ovs_mac_header_len(ovs_key_mac_proto(key));
}

static inline bool ovs_identifier_is_ufid(const struct sw_flow_id *sfid)
{
        return sfid->ufid_len;
}

static inline bool ovs_identifier_is_key(const struct sw_flow_id *sfid)
{
        return !ovs_identifier_is_ufid(sfid);
}

void ovs_flow_stats_update(struct sw_flow *, __be16 tcp_flags,
                           const struct sk_buff *);
void ovs_flow_stats_get(const struct sw_flow *, struct ovs_flow_stats *,
                        unsigned long *used, __be16 *tcp_flags);
void ovs_flow_stats_clear(struct sw_flow *);
u64 ovs_flow_used_time(unsigned long flow_jiffies);

int ovs_flow_key_update(struct sk_buff *skb, struct sw_flow_key *key);
int ovs_flow_key_update_l3l4(struct sk_buff *skb, struct sw_flow_key *key);
int ovs_flow_key_extract(const struct ip_tunnel_info *tun_info,
                         struct sk_buff *skb,
                         struct sw_flow_key *key);
/* Extract key from packet coming from userspace. */
int ovs_flow_key_extract_userspace(struct net *net, const struct nlattr *attr,
                                   struct sk_buff *skb,
                                   struct sw_flow_key *key, bool log);

#endif /* flow.h */
































































   62 




   62 



    5 


   54 


   54 






   51 
    1 



    1 






   24 
   27 




   27 
   23 








   49 




    2 











   41 



   10 


























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        GRE over IPv4 demultiplexer driver
 *
 *        Authors: Dmitry Kozlov (xeb@mail.ru)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/if.h>
#include <linux/icmp.h>
#include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/skbuff.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/netdevice.h>
#include <linux/if_tunnel.h>
#include <linux/spinlock.h>
#include <net/protocol.h>
#include <net/gre.h>
#include <net/erspan.h>

#include <net/icmp.h>
#include <net/route.h>
#include <net/xfrm.h>

static const struct gre_protocol __rcu *gre_proto[GREPROTO_MAX] __read_mostly;

int gre_add_protocol(const struct gre_protocol *proto, u8 version)
{
        if (version >= GREPROTO_MAX)
                return -EINVAL;

        return (cmpxchg((const struct gre_protocol **)&gre_proto[version], NULL, proto) == NULL) ?
                0 : -EBUSY;
}
EXPORT_SYMBOL_GPL(gre_add_protocol);

int gre_del_protocol(const struct gre_protocol *proto, u8 version)
{
        int ret;

        if (version >= GREPROTO_MAX)
                return -EINVAL;

        ret = (cmpxchg((const struct gre_protocol **)&gre_proto[version], proto, NULL) == proto) ?
                0 : -EBUSY;

        if (ret)
                return ret;

        synchronize_rcu();
        return 0;
}
EXPORT_SYMBOL_GPL(gre_del_protocol);

/* Fills in tpi and returns header length to be pulled.
 * Note that caller must use pskb_may_pull() before pulling GRE header.
 */
int gre_parse_header(struct sk_buff *skb, struct tnl_ptk_info *tpi,
                     bool *csum_err, __be16 proto, int nhs)
{
        const struct gre_base_hdr *greh;
        __be32 *options;
        int hdr_len;

        if (unlikely(!pskb_may_pull(skb, nhs + sizeof(struct gre_base_hdr))))
                return -EINVAL;

        greh = (struct gre_base_hdr *)(skb->data + nhs);
        if (unlikely(greh->flags & (GRE_VERSION | GRE_ROUTING)))
                return -EINVAL;

        gre_flags_to_tnl_flags(tpi->flags, greh->flags);
        hdr_len = gre_calc_hlen(tpi->flags);

        if (!pskb_may_pull(skb, nhs + hdr_len))
                return -EINVAL;

        greh = (struct gre_base_hdr *)(skb->data + nhs);
        tpi->proto = greh->protocol;

        options = (__be32 *)(greh + 1);
        if (greh->flags & GRE_CSUM) {
                if (!skb_checksum_simple_validate(skb)) {
                        skb_checksum_try_convert(skb, IPPROTO_GRE,
                                                 null_compute_pseudo);
                } else if (csum_err) {
                        *csum_err = true;
                        return -EINVAL;
                }

                options++;
        }

        if (greh->flags & GRE_KEY) {
                tpi->key = *options;
                options++;
        } else {
                tpi->key = 0;
        }
        if (unlikely(greh->flags & GRE_SEQ)) {
                tpi->seq = *options;
                options++;
        } else {
                tpi->seq = 0;
        }
        /* WCCP version 1 and 2 protocol decoding.
         * - Change protocol to IPv4/IPv6
         * - When dealing with WCCPv2, Skip extra 4 bytes in GRE header
         */
        if (greh->flags == 0 && tpi->proto == htons(ETH_P_WCCP)) {
                u8 _val, *val;

                val = skb_header_pointer(skb, nhs + hdr_len,
                                         sizeof(_val), &_val);
                if (!val)
                        return -EINVAL;
                tpi->proto = proto;
                if ((*val & 0xF0) != 0x40)
                        hdr_len += 4;
        }
        tpi->hdr_len = hdr_len;

        /* ERSPAN ver 1 and 2 protocol sets GRE key field
         * to 0 and sets the configured key in the
         * inner erspan header field
         */
        if ((greh->protocol == htons(ETH_P_ERSPAN) && hdr_len != 4) ||
            greh->protocol == htons(ETH_P_ERSPAN2)) {
                struct erspan_base_hdr *ershdr;

                if (!pskb_may_pull(skb, nhs + hdr_len + sizeof(*ershdr)))
                        return -EINVAL;

                ershdr = (struct erspan_base_hdr *)(skb->data + nhs + hdr_len);
                tpi->key = cpu_to_be32(get_session_id(ershdr));
        }

        return hdr_len;
}
EXPORT_SYMBOL(gre_parse_header);

static int gre_rcv(struct sk_buff *skb)
{
        const struct gre_protocol *proto;
        u8 ver;
        int ret;

        if (!pskb_may_pull(skb, 12))
                goto drop;

        ver = skb->data[1]&0x7f;
        if (ver >= GREPROTO_MAX)
                goto drop;

        rcu_read_lock();
        proto = rcu_dereference(gre_proto[ver]);
        if (!proto || !proto->handler)
                goto drop_nohandler;
        ret = proto->handler(skb);
        rcu_read_unlock();
        return ret;

drop_nohandler:
        rcu_read_unlock();
        dev_core_stats_rx_nohandler_inc(skb->dev);
        kfree_skb(skb);
        return NET_RX_DROP;
drop:
        dev_core_stats_rx_dropped_inc(skb->dev);
        kfree_skb(skb);
        return NET_RX_DROP;
}

static int gre_err(struct sk_buff *skb, u32 info)
{
        const struct gre_protocol *proto;
        const struct iphdr *iph = (const struct iphdr *)skb->data;
        u8 ver = skb->data[(iph->ihl<<2) + 1]&0x7f;
        int err = 0;

        if (ver >= GREPROTO_MAX)
                return -EINVAL;

        rcu_read_lock();
        proto = rcu_dereference(gre_proto[ver]);
        if (proto && proto->err_handler)
                proto->err_handler(skb, info);
        else
                err = -EPROTONOSUPPORT;
        rcu_read_unlock();

        return err;
}

static const struct net_protocol net_gre_protocol = {
        .handler     = gre_rcv,
        .err_handler = gre_err,
};

static int __init gre_init(void)
{
        pr_info("GRE over IPv4 demultiplexer driver\n");

        if (inet_add_protocol(&net_gre_protocol, IPPROTO_GRE) < 0) {
                pr_err("can't add protocol\n");
                return -EAGAIN;
        }
        return 0;
}

static void __exit gre_exit(void)
{
        inet_del_protocol(&net_gre_protocol, IPPROTO_GRE);
}

module_init(gre_init);
module_exit(gre_exit);

MODULE_DESCRIPTION("GRE over IPv4 demultiplexer driver");
MODULE_AUTHOR("D. Kozlov <xeb@mail.ru>");
MODULE_LICENSE("GPL");


































































































































































   37 








































































































































































































































































































































































































































































































































































































































    1 








    1 

















    1 
    1 

















    1 

























































































































    7 










    6 






    1 



    1 










    1 
























































































































































   18 


   14 





   15 

















    1 




    4 










   80 
   35 

   12 


   32 













































































    2 
    2 
















































    5 



















    5 





































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
// SPDX-License-Identifier: GPL-2.0
/* Copyright (C) B.A.T.M.A.N. contributors:
 *
 * Matthias Schiffer
 */

#include "netlink.h"
#include "main.h"

#include <linux/array_size.h>
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/bug.h>
#include <linux/byteorder/generic.h>
#include <linux/cache.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/gfp.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/init.h>
#include <linux/limits.h>
#include <linux/minmax.h>
#include <linux/netdevice.h>
#include <linux/netlink.h>
#include <linux/printk.h>
#include <linux/rtnetlink.h>
#include <linux/skbuff.h>
#include <linux/stddef.h>
#include <linux/types.h>
#include <net/genetlink.h>
#include <net/net_namespace.h>
#include <net/netlink.h>
#include <net/sock.h>
#include <uapi/linux/batadv_packet.h>
#include <uapi/linux/batman_adv.h>

#include "bat_algo.h"
#include "bridge_loop_avoidance.h"
#include "distributed-arp-table.h"
#include "gateway_client.h"
#include "gateway_common.h"
#include "hard-interface.h"
#include "log.h"
#include "mesh-interface.h"
#include "multicast.h"
#include "originator.h"
#include "tp_meter.h"
#include "translation-table.h"

struct genl_family batadv_netlink_family;

/* multicast groups */
enum batadv_netlink_multicast_groups {
        BATADV_NL_MCGRP_CONFIG,
        BATADV_NL_MCGRP_TPMETER,
};

/**
 * enum batadv_genl_ops_flags - flags for genl_ops's internal_flags
 */
enum batadv_genl_ops_flags {
        /**
         * @BATADV_FLAG_NEED_MESH: request requires valid mesh interface in
         *  attribute BATADV_ATTR_MESH_IFINDEX and expects a pointer to it to be
         *  saved in info->user_ptr[0]
         */
        BATADV_FLAG_NEED_MESH = BIT(0),

        /**
         * @BATADV_FLAG_NEED_HARDIF: request requires valid hard interface in
         *  attribute BATADV_ATTR_HARD_IFINDEX and expects a pointer to it to be
         *  saved in info->user_ptr[1]
         */
        BATADV_FLAG_NEED_HARDIF = BIT(1),

        /**
         * @BATADV_FLAG_NEED_VLAN: request requires valid vlan in
         *  attribute BATADV_ATTR_VLANID and expects a pointer to it to be
         *  saved in info->user_ptr[1]
         */
        BATADV_FLAG_NEED_VLAN = BIT(2),
};

static const struct genl_multicast_group batadv_netlink_mcgrps[] = {
        [BATADV_NL_MCGRP_CONFIG] = { .name = BATADV_NL_MCAST_GROUP_CONFIG },
        [BATADV_NL_MCGRP_TPMETER] = { .name = BATADV_NL_MCAST_GROUP_TPMETER },
};

static const struct nla_policy batadv_netlink_policy[NUM_BATADV_ATTR] = {
        [BATADV_ATTR_VERSION]                        = { .type = NLA_STRING },
        [BATADV_ATTR_ALGO_NAME]                        = { .type = NLA_STRING },
        [BATADV_ATTR_MESH_IFINDEX]                = { .type = NLA_U32 },
        [BATADV_ATTR_MESH_IFNAME]                = { .type = NLA_STRING },
        [BATADV_ATTR_MESH_ADDRESS]                = { .len = ETH_ALEN },
        [BATADV_ATTR_HARD_IFINDEX]                = { .type = NLA_U32 },
        [BATADV_ATTR_HARD_IFNAME]                = { .type = NLA_STRING },
        [BATADV_ATTR_HARD_ADDRESS]                = { .len = ETH_ALEN },
        [BATADV_ATTR_ORIG_ADDRESS]                = { .len = ETH_ALEN },
        [BATADV_ATTR_TPMETER_RESULT]                = { .type = NLA_U8 },
        [BATADV_ATTR_TPMETER_TEST_TIME]                = { .type = NLA_U32 },
        [BATADV_ATTR_TPMETER_BYTES]                = { .type = NLA_U64 },
        [BATADV_ATTR_TPMETER_COOKIE]                = { .type = NLA_U32 },
        [BATADV_ATTR_ACTIVE]                        = { .type = NLA_FLAG },
        [BATADV_ATTR_TT_ADDRESS]                = { .len = ETH_ALEN },
        [BATADV_ATTR_TT_TTVN]                        = { .type = NLA_U8 },
        [BATADV_ATTR_TT_LAST_TTVN]                = { .type = NLA_U8 },
        [BATADV_ATTR_TT_CRC32]                        = { .type = NLA_U32 },
        [BATADV_ATTR_TT_VID]                        = { .type = NLA_U16 },
        [BATADV_ATTR_TT_FLAGS]                        = { .type = NLA_U32 },
        [BATADV_ATTR_FLAG_BEST]                        = { .type = NLA_FLAG },
        [BATADV_ATTR_LAST_SEEN_MSECS]                = { .type = NLA_U32 },
        [BATADV_ATTR_NEIGH_ADDRESS]                = { .len = ETH_ALEN },
        [BATADV_ATTR_TQ]                        = { .type = NLA_U8 },
        [BATADV_ATTR_THROUGHPUT]                = { .type = NLA_U32 },
        [BATADV_ATTR_BANDWIDTH_UP]                = { .type = NLA_U32 },
        [BATADV_ATTR_BANDWIDTH_DOWN]                = { .type = NLA_U32 },
        [BATADV_ATTR_ROUTER]                        = { .len = ETH_ALEN },
        [BATADV_ATTR_BLA_OWN]                        = { .type = NLA_FLAG },
        [BATADV_ATTR_BLA_ADDRESS]                = { .len = ETH_ALEN },
        [BATADV_ATTR_BLA_VID]                        = { .type = NLA_U16 },
        [BATADV_ATTR_BLA_BACKBONE]                = { .len = ETH_ALEN },
        [BATADV_ATTR_BLA_CRC]                        = { .type = NLA_U16 },
        [BATADV_ATTR_DAT_CACHE_IP4ADDRESS]        = { .type = NLA_U32 },
        [BATADV_ATTR_DAT_CACHE_HWADDRESS]        = { .len = ETH_ALEN },
        [BATADV_ATTR_DAT_CACHE_VID]                = { .type = NLA_U16 },
        [BATADV_ATTR_MCAST_FLAGS]                = { .type = NLA_U32 },
        [BATADV_ATTR_MCAST_FLAGS_PRIV]                = { .type = NLA_U32 },
        [BATADV_ATTR_VLANID]                        = { .type = NLA_U16 },
        [BATADV_ATTR_AGGREGATED_OGMS_ENABLED]        = { .type = NLA_U8 },
        [BATADV_ATTR_AP_ISOLATION_ENABLED]        = { .type = NLA_U8 },
        [BATADV_ATTR_ISOLATION_MARK]                = { .type = NLA_U32 },
        [BATADV_ATTR_ISOLATION_MASK]                = { .type = NLA_U32 },
        [BATADV_ATTR_BONDING_ENABLED]                = { .type = NLA_U8 },
        [BATADV_ATTR_BRIDGE_LOOP_AVOIDANCE_ENABLED]        = { .type = NLA_U8 },
        [BATADV_ATTR_DISTRIBUTED_ARP_TABLE_ENABLED]        = { .type = NLA_U8 },
        [BATADV_ATTR_FRAGMENTATION_ENABLED]        = { .type = NLA_U8 },
        [BATADV_ATTR_GW_BANDWIDTH_DOWN]                = { .type = NLA_U32 },
        [BATADV_ATTR_GW_BANDWIDTH_UP]                = { .type = NLA_U32 },
        [BATADV_ATTR_GW_MODE]                        = { .type = NLA_U8 },
        [BATADV_ATTR_GW_SEL_CLASS]                = { .type = NLA_U32 },
        [BATADV_ATTR_HOP_PENALTY]                = { .type = NLA_U8 },
        [BATADV_ATTR_LOG_LEVEL]                        = { .type = NLA_U32 },
        [BATADV_ATTR_MULTICAST_FORCEFLOOD_ENABLED]        = { .type = NLA_U8 },
        [BATADV_ATTR_MULTICAST_FANOUT]                = { .type = NLA_U32 },
        [BATADV_ATTR_ORIG_INTERVAL]                = { .type = NLA_U32 },
        [BATADV_ATTR_ELP_INTERVAL]                = { .type = NLA_U32 },
        [BATADV_ATTR_THROUGHPUT_OVERRIDE]        = { .type = NLA_U32 },
};

/**
 * batadv_netlink_get_ifindex() - Extract an interface index from a message
 * @nlh: Message header
 * @attrtype: Attribute which holds an interface index
 *
 * Return: interface index, or 0.
 */
static int batadv_netlink_get_ifindex(const struct nlmsghdr *nlh, int attrtype)
{
        struct nlattr *attr = nlmsg_find_attr(nlh, GENL_HDRLEN, attrtype);

        return (attr && nla_len(attr) == sizeof(u32)) ? nla_get_u32(attr) : 0;
}

/**
 * batadv_netlink_mesh_fill_ap_isolation() - Add ap_isolation meshif attribute
 * @msg: Netlink message to dump into
 * @bat_priv: the bat priv with all the mesh interface information
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_mesh_fill_ap_isolation(struct sk_buff *msg,
                                                 struct batadv_priv *bat_priv)
{
        struct batadv_meshif_vlan *vlan;
        u8 ap_isolation;

        vlan = batadv_meshif_vlan_get(bat_priv, BATADV_NO_FLAGS);
        if (!vlan)
                return 0;

        ap_isolation = atomic_read(&vlan->ap_isolation);
        batadv_meshif_vlan_put(vlan);

        return nla_put_u8(msg, BATADV_ATTR_AP_ISOLATION_ENABLED,
                          !!ap_isolation);
}

/**
 * batadv_netlink_set_mesh_ap_isolation() - Set ap_isolation from genl msg
 * @attr: parsed BATADV_ATTR_AP_ISOLATION_ENABLED attribute
 * @bat_priv: the bat priv with all the mesh interface information
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_set_mesh_ap_isolation(struct nlattr *attr,
                                                struct batadv_priv *bat_priv)
{
        struct batadv_meshif_vlan *vlan;

        vlan = batadv_meshif_vlan_get(bat_priv, BATADV_NO_FLAGS);
        if (!vlan)
                return -ENOENT;

        atomic_set(&vlan->ap_isolation, !!nla_get_u8(attr));
        batadv_meshif_vlan_put(vlan);

        return 0;
}

/**
 * batadv_netlink_mesh_fill() - Fill message with mesh attributes
 * @msg: Netlink message to dump into
 * @bat_priv: the bat priv with all the mesh interface information
 * @cmd: type of message to generate
 * @portid: Port making netlink request
 * @seq: sequence number for message
 * @flags: Additional flags for message
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_mesh_fill(struct sk_buff *msg,
                                    struct batadv_priv *bat_priv,
                                    enum batadv_nl_commands cmd,
                                    u32 portid, u32 seq, int flags)
{
        struct net_device *mesh_iface = bat_priv->mesh_iface;
        struct batadv_hard_iface *primary_if = NULL;
        struct net_device *hard_iface;
        void *hdr;

        hdr = genlmsg_put(msg, portid, seq, &batadv_netlink_family, flags, cmd);
        if (!hdr)
                return -ENOBUFS;

        if (nla_put_string(msg, BATADV_ATTR_VERSION, BATADV_SOURCE_VERSION) ||
            nla_put_string(msg, BATADV_ATTR_ALGO_NAME,
                           bat_priv->algo_ops->name) ||
            nla_put_u32(msg, BATADV_ATTR_MESH_IFINDEX, mesh_iface->ifindex) ||
            nla_put_string(msg, BATADV_ATTR_MESH_IFNAME, mesh_iface->name) ||
            nla_put(msg, BATADV_ATTR_MESH_ADDRESS, ETH_ALEN,
                    mesh_iface->dev_addr) ||
            nla_put_u8(msg, BATADV_ATTR_TT_TTVN,
                       (u8)atomic_read(&bat_priv->tt.vn)))
                goto nla_put_failure;

#ifdef CONFIG_BATMAN_ADV_BLA
        if (nla_put_u16(msg, BATADV_ATTR_BLA_CRC,
                        ntohs(bat_priv->bla.claim_dest.group)))
                goto nla_put_failure;
#endif

        if (batadv_mcast_mesh_info_put(msg, bat_priv))
                goto nla_put_failure;

        primary_if = batadv_primary_if_get_selected(bat_priv);
        if (primary_if && primary_if->if_status == BATADV_IF_ACTIVE) {
                hard_iface = primary_if->net_dev;

                if (nla_put_u32(msg, BATADV_ATTR_HARD_IFINDEX,
                                hard_iface->ifindex) ||
                    nla_put_string(msg, BATADV_ATTR_HARD_IFNAME,
                                   hard_iface->name) ||
                    nla_put(msg, BATADV_ATTR_HARD_ADDRESS, ETH_ALEN,
                            hard_iface->dev_addr))
                        goto nla_put_failure;
        }

        if (nla_put_u8(msg, BATADV_ATTR_AGGREGATED_OGMS_ENABLED,
                       !!atomic_read(&bat_priv->aggregated_ogms)))
                goto nla_put_failure;

        if (batadv_netlink_mesh_fill_ap_isolation(msg, bat_priv))
                goto nla_put_failure;

        if (nla_put_u32(msg, BATADV_ATTR_ISOLATION_MARK,
                        bat_priv->isolation_mark))
                goto nla_put_failure;

        if (nla_put_u32(msg, BATADV_ATTR_ISOLATION_MASK,
                        bat_priv->isolation_mark_mask))
                goto nla_put_failure;

        if (nla_put_u8(msg, BATADV_ATTR_BONDING_ENABLED,
                       !!atomic_read(&bat_priv->bonding)))
                goto nla_put_failure;

#ifdef CONFIG_BATMAN_ADV_BLA
        if (nla_put_u8(msg, BATADV_ATTR_BRIDGE_LOOP_AVOIDANCE_ENABLED,
                       !!atomic_read(&bat_priv->bridge_loop_avoidance)))
                goto nla_put_failure;
#endif /* CONFIG_BATMAN_ADV_BLA */

#ifdef CONFIG_BATMAN_ADV_DAT
        if (nla_put_u8(msg, BATADV_ATTR_DISTRIBUTED_ARP_TABLE_ENABLED,
                       !!atomic_read(&bat_priv->distributed_arp_table)))
                goto nla_put_failure;
#endif /* CONFIG_BATMAN_ADV_DAT */

        if (nla_put_u8(msg, BATADV_ATTR_FRAGMENTATION_ENABLED,
                       !!atomic_read(&bat_priv->fragmentation)))
                goto nla_put_failure;

        if (nla_put_u32(msg, BATADV_ATTR_GW_BANDWIDTH_DOWN,
                        atomic_read(&bat_priv->gw.bandwidth_down)))
                goto nla_put_failure;

        if (nla_put_u32(msg, BATADV_ATTR_GW_BANDWIDTH_UP,
                        atomic_read(&bat_priv->gw.bandwidth_up)))
                goto nla_put_failure;

        if (nla_put_u8(msg, BATADV_ATTR_GW_MODE,
                       atomic_read(&bat_priv->gw.mode)))
                goto nla_put_failure;

        if (bat_priv->algo_ops->gw.get_best_gw_node &&
            bat_priv->algo_ops->gw.is_eligible) {
                /* GW selection class is not available if the routing algorithm
                 * in use does not implement the GW API
                 */
                if (nla_put_u32(msg, BATADV_ATTR_GW_SEL_CLASS,
                                atomic_read(&bat_priv->gw.sel_class)))
                        goto nla_put_failure;
        }

        if (nla_put_u8(msg, BATADV_ATTR_HOP_PENALTY,
                       atomic_read(&bat_priv->hop_penalty)))
                goto nla_put_failure;

#ifdef CONFIG_BATMAN_ADV_DEBUG
        if (nla_put_u32(msg, BATADV_ATTR_LOG_LEVEL,
                        atomic_read(&bat_priv->log_level)))
                goto nla_put_failure;
#endif /* CONFIG_BATMAN_ADV_DEBUG */

#ifdef CONFIG_BATMAN_ADV_MCAST
        if (nla_put_u8(msg, BATADV_ATTR_MULTICAST_FORCEFLOOD_ENABLED,
                       !atomic_read(&bat_priv->multicast_mode)))
                goto nla_put_failure;

        if (nla_put_u32(msg, BATADV_ATTR_MULTICAST_FANOUT,
                        atomic_read(&bat_priv->multicast_fanout)))
                goto nla_put_failure;
#endif /* CONFIG_BATMAN_ADV_MCAST */

        if (nla_put_u32(msg, BATADV_ATTR_ORIG_INTERVAL,
                        atomic_read(&bat_priv->orig_interval)))
                goto nla_put_failure;

        batadv_hardif_put(primary_if);

        genlmsg_end(msg, hdr);
        return 0;

nla_put_failure:
        batadv_hardif_put(primary_if);

        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

/**
 * batadv_netlink_notify_mesh() - send meshif attributes to listener
 * @bat_priv: the bat priv with all the mesh interface information
 *
 * Return: 0 on success, < 0 on error
 */
static int batadv_netlink_notify_mesh(struct batadv_priv *bat_priv)
{
        struct sk_buff *msg;
        int ret;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        ret = batadv_netlink_mesh_fill(msg, bat_priv, BATADV_CMD_SET_MESH,
                                       0, 0, 0);
        if (ret < 0) {
                nlmsg_free(msg);
                return ret;
        }

        genlmsg_multicast_netns(&batadv_netlink_family,
                                dev_net(bat_priv->mesh_iface), msg, 0,
                                BATADV_NL_MCGRP_CONFIG, GFP_KERNEL);

        return 0;
}

/**
 * batadv_netlink_get_mesh() - Get meshif attributes
 * @skb: Netlink message with request data
 * @info: receiver information
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_get_mesh(struct sk_buff *skb, struct genl_info *info)
{
        struct batadv_priv *bat_priv = info->user_ptr[0];
        struct sk_buff *msg;
        int ret;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        ret = batadv_netlink_mesh_fill(msg, bat_priv, BATADV_CMD_GET_MESH,
                                       info->snd_portid, info->snd_seq, 0);
        if (ret < 0) {
                nlmsg_free(msg);
                return ret;
        }

        ret = genlmsg_reply(msg, info);

        return ret;
}

/**
 * batadv_netlink_set_mesh() - Set meshif attributes
 * @skb: Netlink message with request data
 * @info: receiver information
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_set_mesh(struct sk_buff *skb, struct genl_info *info)
{
        struct batadv_priv *bat_priv = info->user_ptr[0];
        struct nlattr *attr;

        if (info->attrs[BATADV_ATTR_AGGREGATED_OGMS_ENABLED]) {
                attr = info->attrs[BATADV_ATTR_AGGREGATED_OGMS_ENABLED];

                atomic_set(&bat_priv->aggregated_ogms, !!nla_get_u8(attr));
        }

        if (info->attrs[BATADV_ATTR_AP_ISOLATION_ENABLED]) {
                attr = info->attrs[BATADV_ATTR_AP_ISOLATION_ENABLED];

                batadv_netlink_set_mesh_ap_isolation(attr, bat_priv);
        }

        if (info->attrs[BATADV_ATTR_ISOLATION_MARK]) {
                attr = info->attrs[BATADV_ATTR_ISOLATION_MARK];

                bat_priv->isolation_mark = nla_get_u32(attr);
        }

        if (info->attrs[BATADV_ATTR_ISOLATION_MASK]) {
                attr = info->attrs[BATADV_ATTR_ISOLATION_MASK];

                bat_priv->isolation_mark_mask = nla_get_u32(attr);
        }

        if (info->attrs[BATADV_ATTR_BONDING_ENABLED]) {
                attr = info->attrs[BATADV_ATTR_BONDING_ENABLED];

                atomic_set(&bat_priv->bonding, !!nla_get_u8(attr));
        }

#ifdef CONFIG_BATMAN_ADV_BLA
        if (info->attrs[BATADV_ATTR_BRIDGE_LOOP_AVOIDANCE_ENABLED]) {
                attr = info->attrs[BATADV_ATTR_BRIDGE_LOOP_AVOIDANCE_ENABLED];

                atomic_set(&bat_priv->bridge_loop_avoidance,
                           !!nla_get_u8(attr));
                batadv_bla_status_update(bat_priv->mesh_iface);
        }
#endif /* CONFIG_BATMAN_ADV_BLA */

#ifdef CONFIG_BATMAN_ADV_DAT
        if (info->attrs[BATADV_ATTR_DISTRIBUTED_ARP_TABLE_ENABLED]) {
                attr = info->attrs[BATADV_ATTR_DISTRIBUTED_ARP_TABLE_ENABLED];

                atomic_set(&bat_priv->distributed_arp_table,
                           !!nla_get_u8(attr));
                batadv_dat_status_update(bat_priv->mesh_iface);
        }
#endif /* CONFIG_BATMAN_ADV_DAT */

        if (info->attrs[BATADV_ATTR_FRAGMENTATION_ENABLED]) {
                attr = info->attrs[BATADV_ATTR_FRAGMENTATION_ENABLED];

                atomic_set(&bat_priv->fragmentation, !!nla_get_u8(attr));

                rtnl_lock();
                batadv_update_min_mtu(bat_priv->mesh_iface);
                rtnl_unlock();
        }

        if (info->attrs[BATADV_ATTR_GW_BANDWIDTH_DOWN]) {
                attr = info->attrs[BATADV_ATTR_GW_BANDWIDTH_DOWN];

                atomic_set(&bat_priv->gw.bandwidth_down, nla_get_u32(attr));
                batadv_gw_tvlv_container_update(bat_priv);
        }

        if (info->attrs[BATADV_ATTR_GW_BANDWIDTH_UP]) {
                attr = info->attrs[BATADV_ATTR_GW_BANDWIDTH_UP];

                atomic_set(&bat_priv->gw.bandwidth_up, nla_get_u32(attr));
                batadv_gw_tvlv_container_update(bat_priv);
        }

        if (info->attrs[BATADV_ATTR_GW_MODE]) {
                u8 gw_mode;

                attr = info->attrs[BATADV_ATTR_GW_MODE];
                gw_mode = nla_get_u8(attr);

                if (gw_mode <= BATADV_GW_MODE_SERVER) {
                        /* Invoking batadv_gw_reselect() is not enough to really
                         * de-select the current GW. It will only instruct the
                         * gateway client code to perform a re-election the next
                         * time that this is needed.
                         *
                         * When gw client mode is being switched off the current
                         * GW must be de-selected explicitly otherwise no GW_ADD
                         * uevent is thrown on client mode re-activation. This
                         * is operation is performed in
                         * batadv_gw_check_client_stop().
                         */
                        batadv_gw_reselect(bat_priv);

                        /* always call batadv_gw_check_client_stop() before
                         * changing the gateway state
                         */
                        batadv_gw_check_client_stop(bat_priv);
                        atomic_set(&bat_priv->gw.mode, gw_mode);
                        batadv_gw_tvlv_container_update(bat_priv);
                }
        }

        if (info->attrs[BATADV_ATTR_GW_SEL_CLASS] &&
            bat_priv->algo_ops->gw.get_best_gw_node &&
            bat_priv->algo_ops->gw.is_eligible) {
                /* setting the GW selection class is allowed only if the routing
                 * algorithm in use implements the GW API
                 */

                u32 sel_class_max = bat_priv->algo_ops->gw.sel_class_max;
                u32 sel_class;

                attr = info->attrs[BATADV_ATTR_GW_SEL_CLASS];
                sel_class = nla_get_u32(attr);

                if (sel_class >= 1 && sel_class <= sel_class_max) {
                        atomic_set(&bat_priv->gw.sel_class, sel_class);
                        batadv_gw_reselect(bat_priv);
                }
        }

        if (info->attrs[BATADV_ATTR_HOP_PENALTY]) {
                attr = info->attrs[BATADV_ATTR_HOP_PENALTY];

                atomic_set(&bat_priv->hop_penalty, nla_get_u8(attr));
        }

#ifdef CONFIG_BATMAN_ADV_DEBUG
        if (info->attrs[BATADV_ATTR_LOG_LEVEL]) {
                attr = info->attrs[BATADV_ATTR_LOG_LEVEL];

                atomic_set(&bat_priv->log_level,
                           nla_get_u32(attr) & BATADV_DBG_ALL);
        }
#endif /* CONFIG_BATMAN_ADV_DEBUG */

#ifdef CONFIG_BATMAN_ADV_MCAST
        if (info->attrs[BATADV_ATTR_MULTICAST_FORCEFLOOD_ENABLED]) {
                attr = info->attrs[BATADV_ATTR_MULTICAST_FORCEFLOOD_ENABLED];

                atomic_set(&bat_priv->multicast_mode, !nla_get_u8(attr));
        }

        if (info->attrs[BATADV_ATTR_MULTICAST_FANOUT]) {
                attr = info->attrs[BATADV_ATTR_MULTICAST_FANOUT];

                atomic_set(&bat_priv->multicast_fanout, nla_get_u32(attr));
        }
#endif /* CONFIG_BATMAN_ADV_MCAST */

        if (info->attrs[BATADV_ATTR_ORIG_INTERVAL]) {
                u32 orig_interval;

                attr = info->attrs[BATADV_ATTR_ORIG_INTERVAL];
                orig_interval = nla_get_u32(attr);

                orig_interval = min_t(u32, orig_interval, INT_MAX);
                orig_interval = max_t(u32, orig_interval, 2 * BATADV_JITTER);

                atomic_set(&bat_priv->orig_interval, orig_interval);
        }

        batadv_netlink_notify_mesh(bat_priv);

        return 0;
}

/**
 * batadv_netlink_tp_meter_put() - Fill information of started tp_meter session
 * @msg: netlink message to be sent back
 * @cookie: tp meter session cookie
 *
 *  Return: 0 on success, < 0 on error
 */
static int
batadv_netlink_tp_meter_put(struct sk_buff *msg, u32 cookie)
{
        if (nla_put_u32(msg, BATADV_ATTR_TPMETER_COOKIE, cookie))
                return -ENOBUFS;

        return 0;
}

/**
 * batadv_netlink_tpmeter_notify() - send tp_meter result via netlink to client
 * @bat_priv: the bat priv with all the mesh interface information
 * @dst: destination of tp_meter session
 * @result: reason for tp meter session stop
 * @test_time: total time of the tp_meter session
 * @total_bytes: bytes acked to the receiver
 * @cookie: cookie of tp_meter session
 *
 * Return: 0 on success, < 0 on error
 */
int batadv_netlink_tpmeter_notify(struct batadv_priv *bat_priv, const u8 *dst,
                                  u8 result, u32 test_time, u64 total_bytes,
                                  u32 cookie)
{
        struct sk_buff *msg;
        void *hdr;
        int ret;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        hdr = genlmsg_put(msg, 0, 0, &batadv_netlink_family, 0,
                          BATADV_CMD_TP_METER);
        if (!hdr) {
                ret = -ENOBUFS;
                goto err_genlmsg;
        }

        if (nla_put_u32(msg, BATADV_ATTR_TPMETER_COOKIE, cookie))
                goto nla_put_failure;

        if (nla_put_u32(msg, BATADV_ATTR_TPMETER_TEST_TIME, test_time))
                goto nla_put_failure;

        if (nla_put_u64_64bit(msg, BATADV_ATTR_TPMETER_BYTES, total_bytes,
                              BATADV_ATTR_PAD))
                goto nla_put_failure;

        if (nla_put_u8(msg, BATADV_ATTR_TPMETER_RESULT, result))
                goto nla_put_failure;

        if (nla_put(msg, BATADV_ATTR_ORIG_ADDRESS, ETH_ALEN, dst))
                goto nla_put_failure;

        genlmsg_end(msg, hdr);

        genlmsg_multicast_netns(&batadv_netlink_family,
                                dev_net(bat_priv->mesh_iface), msg, 0,
                                BATADV_NL_MCGRP_TPMETER, GFP_KERNEL);

        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        ret = -EMSGSIZE;

err_genlmsg:
        nlmsg_free(msg);
        return ret;
}

/**
 * batadv_netlink_tp_meter_start() - Start a new tp_meter session
 * @skb: received netlink message
 * @info: receiver information
 *
 * Return: 0 on success, < 0 on error
 */
static int
batadv_netlink_tp_meter_start(struct sk_buff *skb, struct genl_info *info)
{
        struct batadv_priv *bat_priv = info->user_ptr[0];
        struct sk_buff *msg = NULL;
        u32 test_length;
        void *msg_head;
        u32 cookie;
        u8 *dst;
        int ret;

        if (!info->attrs[BATADV_ATTR_ORIG_ADDRESS])
                return -EINVAL;

        if (!info->attrs[BATADV_ATTR_TPMETER_TEST_TIME])
                return -EINVAL;

        dst = nla_data(info->attrs[BATADV_ATTR_ORIG_ADDRESS]);

        test_length = nla_get_u32(info->attrs[BATADV_ATTR_TPMETER_TEST_TIME]);

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg) {
                ret = -ENOMEM;
                goto out;
        }

        msg_head = genlmsg_put(msg, info->snd_portid, info->snd_seq,
                               &batadv_netlink_family, 0,
                               BATADV_CMD_TP_METER);
        if (!msg_head) {
                ret = -ENOBUFS;
                goto out;
        }

        batadv_tp_start(bat_priv, dst, test_length, &cookie);

        ret = batadv_netlink_tp_meter_put(msg, cookie);

 out:
        if (ret) {
                if (msg)
                        nlmsg_free(msg);
                return ret;
        }

        genlmsg_end(msg, msg_head);
        return genlmsg_reply(msg, info);
}

/**
 * batadv_netlink_tp_meter_cancel() - Cancel a running tp_meter session
 * @skb: received netlink message
 * @info: receiver information
 *
 * Return: 0 on success, < 0 on error
 */
static int
batadv_netlink_tp_meter_cancel(struct sk_buff *skb, struct genl_info *info)
{
        struct batadv_priv *bat_priv = info->user_ptr[0];
        u8 *dst;
        int ret = 0;

        if (!info->attrs[BATADV_ATTR_ORIG_ADDRESS])
                return -EINVAL;

        dst = nla_data(info->attrs[BATADV_ATTR_ORIG_ADDRESS]);

        batadv_tp_stop(bat_priv, dst, BATADV_TP_REASON_CANCEL);

        return ret;
}

/**
 * batadv_netlink_hardif_fill() - Fill message with hardif attributes
 * @msg: Netlink message to dump into
 * @bat_priv: the bat priv with all the mesh interface information
 * @hard_iface: hard interface which was modified
 * @cmd: type of message to generate
 * @portid: Port making netlink request
 * @seq: sequence number for message
 * @flags: Additional flags for message
 * @cb: Control block containing additional options
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_hardif_fill(struct sk_buff *msg,
                                      struct batadv_priv *bat_priv,
                                      struct batadv_hard_iface *hard_iface,
                                      enum batadv_nl_commands cmd,
                                      u32 portid, u32 seq, int flags,
                                      struct netlink_callback *cb)
{
        struct net_device *net_dev = hard_iface->net_dev;
        void *hdr;

        hdr = genlmsg_put(msg, portid, seq, &batadv_netlink_family, flags, cmd);
        if (!hdr)
                return -ENOBUFS;

        if (cb)
                genl_dump_check_consistent(cb, hdr);

        if (nla_put_u32(msg, BATADV_ATTR_MESH_IFINDEX,
                        bat_priv->mesh_iface->ifindex))
                goto nla_put_failure;

        if (nla_put_string(msg, BATADV_ATTR_MESH_IFNAME,
                           bat_priv->mesh_iface->name))
                goto nla_put_failure;

        if (nla_put_u32(msg, BATADV_ATTR_HARD_IFINDEX,
                        net_dev->ifindex) ||
            nla_put_string(msg, BATADV_ATTR_HARD_IFNAME,
                           net_dev->name) ||
            nla_put(msg, BATADV_ATTR_HARD_ADDRESS, ETH_ALEN,
                    net_dev->dev_addr))
                goto nla_put_failure;

        if (hard_iface->if_status == BATADV_IF_ACTIVE) {
                if (nla_put_flag(msg, BATADV_ATTR_ACTIVE))
                        goto nla_put_failure;
        }

        if (nla_put_u8(msg, BATADV_ATTR_HOP_PENALTY,
                       atomic_read(&hard_iface->hop_penalty)))
                goto nla_put_failure;

#ifdef CONFIG_BATMAN_ADV_BATMAN_V
        if (nla_put_u32(msg, BATADV_ATTR_ELP_INTERVAL,
                        atomic_read(&hard_iface->bat_v.elp_interval)))
                goto nla_put_failure;

        if (nla_put_u32(msg, BATADV_ATTR_THROUGHPUT_OVERRIDE,
                        atomic_read(&hard_iface->bat_v.throughput_override)))
                goto nla_put_failure;
#endif /* CONFIG_BATMAN_ADV_BATMAN_V */

        genlmsg_end(msg, hdr);
        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

/**
 * batadv_netlink_notify_hardif() - send hardif attributes to listener
 * @bat_priv: the bat priv with all the mesh interface information
 * @hard_iface: hard interface which was modified
 *
 * Return: 0 on success, < 0 on error
 */
static int batadv_netlink_notify_hardif(struct batadv_priv *bat_priv,
                                        struct batadv_hard_iface *hard_iface)
{
        struct sk_buff *msg;
        int ret;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        ret = batadv_netlink_hardif_fill(msg, bat_priv, hard_iface,
                                         BATADV_CMD_SET_HARDIF, 0, 0, 0, NULL);
        if (ret < 0) {
                nlmsg_free(msg);
                return ret;
        }

        genlmsg_multicast_netns(&batadv_netlink_family,
                                dev_net(bat_priv->mesh_iface), msg, 0,
                                BATADV_NL_MCGRP_CONFIG, GFP_KERNEL);

        return 0;
}

/**
 * batadv_netlink_cmd_get_hardif() - Get hardif attributes
 * @skb: Netlink message with request data
 * @info: receiver information
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_cmd_get_hardif(struct sk_buff *skb,
                                         struct genl_info *info)
{
        struct batadv_hard_iface *hard_iface = info->user_ptr[1];
        struct batadv_priv *bat_priv = info->user_ptr[0];
        struct sk_buff *msg;
        int ret;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        ret = batadv_netlink_hardif_fill(msg, bat_priv, hard_iface,
                                         BATADV_CMD_GET_HARDIF,
                                         info->snd_portid, info->snd_seq, 0,
                                         NULL);
        if (ret < 0) {
                nlmsg_free(msg);
                return ret;
        }

        ret = genlmsg_reply(msg, info);

        return ret;
}

/**
 * batadv_netlink_set_hardif() - Set hardif attributes
 * @skb: Netlink message with request data
 * @info: receiver information
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_set_hardif(struct sk_buff *skb,
                                     struct genl_info *info)
{
        struct batadv_hard_iface *hard_iface = info->user_ptr[1];
        struct batadv_priv *bat_priv = info->user_ptr[0];
        struct nlattr *attr;

        if (info->attrs[BATADV_ATTR_HOP_PENALTY]) {
                attr = info->attrs[BATADV_ATTR_HOP_PENALTY];

                atomic_set(&hard_iface->hop_penalty, nla_get_u8(attr));
        }

#ifdef CONFIG_BATMAN_ADV_BATMAN_V

        if (info->attrs[BATADV_ATTR_ELP_INTERVAL]) {
                attr = info->attrs[BATADV_ATTR_ELP_INTERVAL];

                atomic_set(&hard_iface->bat_v.elp_interval, nla_get_u32(attr));
        }

        if (info->attrs[BATADV_ATTR_THROUGHPUT_OVERRIDE]) {
                attr = info->attrs[BATADV_ATTR_THROUGHPUT_OVERRIDE];

                atomic_set(&hard_iface->bat_v.throughput_override,
                           nla_get_u32(attr));
        }
#endif /* CONFIG_BATMAN_ADV_BATMAN_V */

        batadv_netlink_notify_hardif(bat_priv, hard_iface);

        return 0;
}

/**
 * batadv_netlink_dump_hardif() - Dump all hard interface into a messages
 * @msg: Netlink message to dump into
 * @cb: Parameters from query
 *
 * Return: error code, or length of reply message on success
 */
static int
batadv_netlink_dump_hardif(struct sk_buff *msg, struct netlink_callback *cb)
{
        struct net_device *mesh_iface;
        struct batadv_hard_iface *hard_iface;
        struct batadv_priv *bat_priv;
        int portid = NETLINK_CB(cb->skb).portid;
        int skip = cb->args[0];
        struct list_head *iter;
        int i = 0;

        mesh_iface = batadv_netlink_get_meshif(cb);
        if (IS_ERR(mesh_iface))
                return PTR_ERR(mesh_iface);

        bat_priv = netdev_priv(mesh_iface);

        rtnl_lock();
        cb->seq = batadv_hardif_generation << 1 | 1;

        netdev_for_each_lower_private(mesh_iface, hard_iface, iter) {
                if (i++ < skip)
                        continue;

                if (batadv_netlink_hardif_fill(msg, bat_priv, hard_iface,
                                               BATADV_CMD_GET_HARDIF,
                                               portid, cb->nlh->nlmsg_seq,
                                               NLM_F_MULTI, cb)) {
                        i--;
                        break;
                }
        }

        rtnl_unlock();

        dev_put(mesh_iface);

        cb->args[0] = i;

        return msg->len;
}

/**
 * batadv_netlink_vlan_fill() - Fill message with vlan attributes
 * @msg: Netlink message to dump into
 * @bat_priv: the bat priv with all the mesh interface information
 * @vlan: vlan which was modified
 * @cmd: type of message to generate
 * @portid: Port making netlink request
 * @seq: sequence number for message
 * @flags: Additional flags for message
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_vlan_fill(struct sk_buff *msg,
                                    struct batadv_priv *bat_priv,
                                    struct batadv_meshif_vlan *vlan,
                                    enum batadv_nl_commands cmd,
                                    u32 portid, u32 seq, int flags)
{
        void *hdr;

        hdr = genlmsg_put(msg, portid, seq, &batadv_netlink_family, flags, cmd);
        if (!hdr)
                return -ENOBUFS;

        if (nla_put_u32(msg, BATADV_ATTR_MESH_IFINDEX,
                        bat_priv->mesh_iface->ifindex))
                goto nla_put_failure;

        if (nla_put_string(msg, BATADV_ATTR_MESH_IFNAME,
                           bat_priv->mesh_iface->name))
                goto nla_put_failure;

        if (nla_put_u32(msg, BATADV_ATTR_VLANID, vlan->vid & VLAN_VID_MASK))
                goto nla_put_failure;

        if (nla_put_u8(msg, BATADV_ATTR_AP_ISOLATION_ENABLED,
                       !!atomic_read(&vlan->ap_isolation)))
                goto nla_put_failure;

        genlmsg_end(msg, hdr);
        return 0;

nla_put_failure:
        genlmsg_cancel(msg, hdr);
        return -EMSGSIZE;
}

/**
 * batadv_netlink_notify_vlan() - send vlan attributes to listener
 * @bat_priv: the bat priv with all the mesh interface information
 * @vlan: vlan which was modified
 *
 * Return: 0 on success, < 0 on error
 */
static int batadv_netlink_notify_vlan(struct batadv_priv *bat_priv,
                                      struct batadv_meshif_vlan *vlan)
{
        struct sk_buff *msg;
        int ret;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        ret = batadv_netlink_vlan_fill(msg, bat_priv, vlan,
                                       BATADV_CMD_SET_VLAN, 0, 0, 0);
        if (ret < 0) {
                nlmsg_free(msg);
                return ret;
        }

        genlmsg_multicast_netns(&batadv_netlink_family,
                                dev_net(bat_priv->mesh_iface), msg, 0,
                                BATADV_NL_MCGRP_CONFIG, GFP_KERNEL);

        return 0;
}

/**
 * batadv_netlink_get_vlan() - Get vlan attributes
 * @skb: Netlink message with request data
 * @info: receiver information
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_get_vlan(struct sk_buff *skb, struct genl_info *info)
{
        struct batadv_meshif_vlan *vlan = info->user_ptr[1];
        struct batadv_priv *bat_priv = info->user_ptr[0];
        struct sk_buff *msg;
        int ret;

        msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!msg)
                return -ENOMEM;

        ret = batadv_netlink_vlan_fill(msg, bat_priv, vlan, BATADV_CMD_GET_VLAN,
                                       info->snd_portid, info->snd_seq, 0);
        if (ret < 0) {
                nlmsg_free(msg);
                return ret;
        }

        ret = genlmsg_reply(msg, info);

        return ret;
}

/**
 * batadv_netlink_set_vlan() - Get vlan attributes
 * @skb: Netlink message with request data
 * @info: receiver information
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_netlink_set_vlan(struct sk_buff *skb, struct genl_info *info)
{
        struct batadv_meshif_vlan *vlan = info->user_ptr[1];
        struct batadv_priv *bat_priv = info->user_ptr[0];
        struct nlattr *attr;

        if (info->attrs[BATADV_ATTR_AP_ISOLATION_ENABLED]) {
                attr = info->attrs[BATADV_ATTR_AP_ISOLATION_ENABLED];

                atomic_set(&vlan->ap_isolation, !!nla_get_u8(attr));
        }

        batadv_netlink_notify_vlan(bat_priv, vlan);

        return 0;
}

/**
 * batadv_netlink_get_meshif_from_ifindex() - Get mesh-iface from ifindex
 * @net: the applicable net namespace
 * @ifindex: index of the mesh interface
 *
 * Return: Pointer to mesh interface (with increased refcnt) on success, error
 *  pointer on error
 */
static struct net_device *
batadv_netlink_get_meshif_from_ifindex(struct net *net, int ifindex)
{
        struct net_device *mesh_iface;

        mesh_iface = dev_get_by_index(net, ifindex);
        if (!mesh_iface)
                return ERR_PTR(-ENODEV);

        if (!batadv_meshif_is_valid(mesh_iface))
                goto err_put_meshif;

        return mesh_iface;

err_put_meshif:
        dev_put(mesh_iface);

        return ERR_PTR(-EINVAL);
}

/**
 * batadv_netlink_get_meshif_from_info() - Get mesh-iface from genl attributes
 * @net: the applicable net namespace
 * @info: receiver information
 *
 * Return: Pointer to mesh interface (with increased refcnt) on success, error
 *  pointer on error
 */
static struct net_device *
batadv_netlink_get_meshif_from_info(struct net *net, struct genl_info *info)
{
        int ifindex;

        if (!info->attrs[BATADV_ATTR_MESH_IFINDEX])
                return ERR_PTR(-EINVAL);

        ifindex = nla_get_u32(info->attrs[BATADV_ATTR_MESH_IFINDEX]);

        return batadv_netlink_get_meshif_from_ifindex(net, ifindex);
}

/**
 * batadv_netlink_get_meshif() - Retrieve mesh interface from netlink callback
 * @cb: callback structure containing arguments
 *
 * Return: Pointer to mesh interface (with increased refcnt) on success, error
 *  pointer on error
 */
struct net_device *batadv_netlink_get_meshif(struct netlink_callback *cb)
{
        int ifindex = batadv_netlink_get_ifindex(cb->nlh,
                                                 BATADV_ATTR_MESH_IFINDEX);
        if (!ifindex)
                return ERR_PTR(-ENONET);

        return batadv_netlink_get_meshif_from_ifindex(sock_net(cb->skb->sk),
                                                      ifindex);
}

/**
 * batadv_netlink_get_hardif_from_ifindex() - Get hard-iface from ifindex
 * @bat_priv: the bat priv with all the mesh interface information
 * @net: the applicable net namespace
 * @ifindex: index of the hard interface
 *
 * Return: Pointer to hard interface (with increased refcnt) on success, error
 *  pointer on error
 */
static struct batadv_hard_iface *
batadv_netlink_get_hardif_from_ifindex(struct batadv_priv *bat_priv,
                                       struct net *net, int ifindex)
{
        struct batadv_hard_iface *hard_iface;
        struct net_device *hard_dev;

        hard_dev = dev_get_by_index(net, ifindex);
        if (!hard_dev)
                return ERR_PTR(-ENODEV);

        hard_iface = batadv_hardif_get_by_netdev(hard_dev);
        if (!hard_iface)
                goto err_put_harddev;

        if (hard_iface->mesh_iface != bat_priv->mesh_iface)
                goto err_put_hardif;

        /* hard_dev is referenced by hard_iface and not needed here */
        dev_put(hard_dev);

        return hard_iface;

err_put_hardif:
        batadv_hardif_put(hard_iface);
err_put_harddev:
        dev_put(hard_dev);

        return ERR_PTR(-EINVAL);
}

/**
 * batadv_netlink_get_hardif_from_info() - Get hard-iface from genl attributes
 * @bat_priv: the bat priv with all the mesh interface information
 * @net: the applicable net namespace
 * @info: receiver information
 *
 * Return: Pointer to hard interface (with increased refcnt) on success, error
 *  pointer on error
 */
static struct batadv_hard_iface *
batadv_netlink_get_hardif_from_info(struct batadv_priv *bat_priv,
                                    struct net *net, struct genl_info *info)
{
        int ifindex;

        if (!info->attrs[BATADV_ATTR_HARD_IFINDEX])
                return ERR_PTR(-EINVAL);

        ifindex = nla_get_u32(info->attrs[BATADV_ATTR_HARD_IFINDEX]);

        return batadv_netlink_get_hardif_from_ifindex(bat_priv, net, ifindex);
}

/**
 * batadv_netlink_get_hardif() - Retrieve hard interface from netlink callback
 * @bat_priv: the bat priv with all the mesh interface information
 * @cb: callback structure containing arguments
 *
 * Return: Pointer to hard interface (with increased refcnt) on success, error
 *  pointer on error
 */
struct batadv_hard_iface *
batadv_netlink_get_hardif(struct batadv_priv *bat_priv,
                          struct netlink_callback *cb)
{
        int ifindex = batadv_netlink_get_ifindex(cb->nlh,
                                                 BATADV_ATTR_HARD_IFINDEX);
        if (!ifindex)
                return ERR_PTR(-ENONET);

        return batadv_netlink_get_hardif_from_ifindex(bat_priv,
                                                      sock_net(cb->skb->sk),
                                                      ifindex);
}

/**
 * batadv_get_vlan_from_info() - Retrieve vlan from genl attributes
 * @bat_priv: the bat priv with all the mesh interface information
 * @net: the applicable net namespace
 * @info: receiver information
 *
 * Return: Pointer to vlan on success (with increased refcnt), error pointer
 *  on error
 */
static struct batadv_meshif_vlan *
batadv_get_vlan_from_info(struct batadv_priv *bat_priv, struct net *net,
                          struct genl_info *info)
{
        struct batadv_meshif_vlan *vlan;
        u16 vid;

        if (!info->attrs[BATADV_ATTR_VLANID])
                return ERR_PTR(-EINVAL);

        vid = nla_get_u16(info->attrs[BATADV_ATTR_VLANID]);

        vlan = batadv_meshif_vlan_get(bat_priv, vid | BATADV_VLAN_HAS_TAG);
        if (!vlan)
                return ERR_PTR(-ENOENT);

        return vlan;
}

/**
 * batadv_pre_doit() - Prepare batman-adv genl doit request
 * @ops: requested netlink operation
 * @skb: Netlink message with request data
 * @info: receiver information
 *
 * Return: 0 on success or negative error number in case of failure
 */
static int batadv_pre_doit(const struct genl_split_ops *ops,
                           struct sk_buff *skb,
                           struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct batadv_hard_iface *hard_iface;
        struct batadv_priv *bat_priv = NULL;
        struct batadv_meshif_vlan *vlan;
        struct net_device *mesh_iface;
        u8 user_ptr1_flags;
        u8 mesh_dep_flags;
        int ret;

        user_ptr1_flags = BATADV_FLAG_NEED_HARDIF | BATADV_FLAG_NEED_VLAN;
        if (WARN_ON(hweight8(ops->internal_flags & user_ptr1_flags) > 1))
                return -EINVAL;

        mesh_dep_flags = BATADV_FLAG_NEED_HARDIF | BATADV_FLAG_NEED_VLAN;
        if (WARN_ON((ops->internal_flags & mesh_dep_flags) &&
                    (~ops->internal_flags & BATADV_FLAG_NEED_MESH)))
                return -EINVAL;

        if (ops->internal_flags & BATADV_FLAG_NEED_MESH) {
                mesh_iface = batadv_netlink_get_meshif_from_info(net, info);
                if (IS_ERR(mesh_iface))
                        return PTR_ERR(mesh_iface);

                bat_priv = netdev_priv(mesh_iface);
                info->user_ptr[0] = bat_priv;
        }

        if (ops->internal_flags & BATADV_FLAG_NEED_HARDIF) {
                hard_iface = batadv_netlink_get_hardif_from_info(bat_priv, net,
                                                                 info);
                if (IS_ERR(hard_iface)) {
                        ret = PTR_ERR(hard_iface);
                        goto err_put_meshif;
                }

                info->user_ptr[1] = hard_iface;
        }

        if (ops->internal_flags & BATADV_FLAG_NEED_VLAN) {
                vlan = batadv_get_vlan_from_info(bat_priv, net, info);
                if (IS_ERR(vlan)) {
                        ret = PTR_ERR(vlan);
                        goto err_put_meshif;
                }

                info->user_ptr[1] = vlan;
        }

        return 0;

err_put_meshif:
        if (bat_priv)
                dev_put(bat_priv->mesh_iface);

        return ret;
}

/**
 * batadv_post_doit() - End batman-adv genl doit request
 * @ops: requested netlink operation
 * @skb: Netlink message with request data
 * @info: receiver information
 */
static void batadv_post_doit(const struct genl_split_ops *ops,
                             struct sk_buff *skb,
                             struct genl_info *info)
{
        struct batadv_hard_iface *hard_iface;
        struct batadv_meshif_vlan *vlan;
        struct batadv_priv *bat_priv;

        if (ops->internal_flags & BATADV_FLAG_NEED_HARDIF &&
            info->user_ptr[1]) {
                hard_iface = info->user_ptr[1];

                batadv_hardif_put(hard_iface);
        }

        if (ops->internal_flags & BATADV_FLAG_NEED_VLAN && info->user_ptr[1]) {
                vlan = info->user_ptr[1];
                batadv_meshif_vlan_put(vlan);
        }

        if (ops->internal_flags & BATADV_FLAG_NEED_MESH && info->user_ptr[0]) {
                bat_priv = info->user_ptr[0];
                dev_put(bat_priv->mesh_iface);
        }
}

static const struct genl_small_ops batadv_netlink_ops[] = {
        {
                .cmd = BATADV_CMD_GET_MESH,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                /* can be retrieved by unprivileged users */
                .doit = batadv_netlink_get_mesh,
                .internal_flags = BATADV_FLAG_NEED_MESH,
        },
        {
                .cmd = BATADV_CMD_TP_METER,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .doit = batadv_netlink_tp_meter_start,
                .internal_flags = BATADV_FLAG_NEED_MESH,
        },
        {
                .cmd = BATADV_CMD_TP_METER_CANCEL,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .doit = batadv_netlink_tp_meter_cancel,
                .internal_flags = BATADV_FLAG_NEED_MESH,
        },
        {
                .cmd = BATADV_CMD_GET_ROUTING_ALGOS,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_algo_dump,
        },
        {
                .cmd = BATADV_CMD_GET_HARDIF,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                /* can be retrieved by unprivileged users */
                .dumpit = batadv_netlink_dump_hardif,
                .doit = batadv_netlink_cmd_get_hardif,
                .internal_flags = BATADV_FLAG_NEED_MESH |
                                  BATADV_FLAG_NEED_HARDIF,
        },
        {
                .cmd = BATADV_CMD_GET_TRANSTABLE_LOCAL,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_tt_local_dump,
        },
        {
                .cmd = BATADV_CMD_GET_TRANSTABLE_GLOBAL,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_tt_global_dump,
        },
        {
                .cmd = BATADV_CMD_GET_ORIGINATORS,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_orig_dump,
        },
        {
                .cmd = BATADV_CMD_GET_NEIGHBORS,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_hardif_neigh_dump,
        },
        {
                .cmd = BATADV_CMD_GET_GATEWAYS,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_gw_dump,
        },
        {
                .cmd = BATADV_CMD_GET_BLA_CLAIM,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_bla_claim_dump,
        },
        {
                .cmd = BATADV_CMD_GET_BLA_BACKBONE,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_bla_backbone_dump,
        },
        {
                .cmd = BATADV_CMD_GET_DAT_CACHE,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_dat_cache_dump,
        },
        {
                .cmd = BATADV_CMD_GET_MCAST_FLAGS,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .dumpit = batadv_mcast_flags_dump,
        },
        {
                .cmd = BATADV_CMD_SET_MESH,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .doit = batadv_netlink_set_mesh,
                .internal_flags = BATADV_FLAG_NEED_MESH,
        },
        {
                .cmd = BATADV_CMD_SET_HARDIF,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .doit = batadv_netlink_set_hardif,
                .internal_flags = BATADV_FLAG_NEED_MESH |
                                  BATADV_FLAG_NEED_HARDIF,
        },
        {
                .cmd = BATADV_CMD_GET_VLAN,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                /* can be retrieved by unprivileged users */
                .doit = batadv_netlink_get_vlan,
                .internal_flags = BATADV_FLAG_NEED_MESH |
                                  BATADV_FLAG_NEED_VLAN,
        },
        {
                .cmd = BATADV_CMD_SET_VLAN,
                .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
                .flags = GENL_UNS_ADMIN_PERM,
                .doit = batadv_netlink_set_vlan,
                .internal_flags = BATADV_FLAG_NEED_MESH |
                                  BATADV_FLAG_NEED_VLAN,
        },
};

struct genl_family batadv_netlink_family __ro_after_init = {
        .hdrsize = 0,
        .name = BATADV_NL_NAME,
        .version = 1,
        .maxattr = BATADV_ATTR_MAX,
        .policy = batadv_netlink_policy,
        .netnsok = true,
        .pre_doit = batadv_pre_doit,
        .post_doit = batadv_post_doit,
        .module = THIS_MODULE,
        .small_ops = batadv_netlink_ops,
        .n_small_ops = ARRAY_SIZE(batadv_netlink_ops),
        .resv_start_op = BATADV_CMD_SET_VLAN + 1,
        .mcgrps = batadv_netlink_mcgrps,
        .n_mcgrps = ARRAY_SIZE(batadv_netlink_mcgrps),
};

/**
 * batadv_netlink_register() - register batadv genl netlink family
 */
void __init batadv_netlink_register(void)
{
        int ret;

        ret = genl_register_family(&batadv_netlink_family);
        if (ret)
                pr_warn("unable to register netlink family\n");
}

/**
 * batadv_netlink_unregister() - unregister batadv genl netlink family
 */
void batadv_netlink_unregister(void)
{
        genl_unregister_family(&batadv_netlink_family);
}
















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    4 



   16 





    4 























   10 









   54 








    3 





   60 































   59 
    1 





   59 











   87 







   20 

   64 

    4 



   17 






   22 

   44 











   53 


    7 

    2 











    4 






























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C)2002 USAGI/WIDE Project
 *
 * Authors
 *
 *        Mitsuru KANDA @USAGI       : IPv6 Support
 *        Kazunori MIYAZAWA @USAGI   :
 *        Kunihiro Ishiguro <kunihiro@ipinfusion.com>
 *
 *        This file is derived from net/ipv4/esp.c
 */

#define pr_fmt(fmt) "IPv6: " fmt

#include <crypto/aead.h>
#include <crypto/authenc.h>
#include <linux/err.h>
#include <linux/module.h>
#include <net/ip.h>
#include <net/xfrm.h>
#include <net/esp.h>
#include <linux/scatterlist.h>
#include <linux/kernel.h>
#include <linux/pfkeyv2.h>
#include <linux/random.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <net/ip6_checksum.h>
#include <net/ip6_route.h>
#include <net/icmp.h>
#include <net/ipv6.h>
#include <net/protocol.h>
#include <net/udp.h>
#include <linux/icmpv6.h>
#include <net/tcp.h>
#include <net/espintcp.h>
#include <net/inet6_hashtables.h>
#include <linux/skbuff_ref.h>

#include <linux/highmem.h>

struct esp_skb_cb {
        struct xfrm_skb_cb xfrm;
        void *tmp;
};

struct esp_output_extra {
        __be32 seqhi;
        u32 esphoff;
};

#define ESP_SKB_CB(__skb) ((struct esp_skb_cb *)&((__skb)->cb[0]))

/*
 * Allocate an AEAD request structure with extra space for SG and IV.
 *
 * For alignment considerations the upper 32 bits of the sequence number are
 * placed at the front, if present. Followed by the IV, the request and finally
 * the SG list.
 *
 * TODO: Use spare space in skb for this where possible.
 */
static void *esp_alloc_tmp(struct crypto_aead *aead, int nfrags, int seqihlen)
{
        unsigned int len;

        len = seqihlen;

        len += crypto_aead_ivsize(aead);

        if (len) {
                len += crypto_aead_alignmask(aead) &
                       ~(crypto_tfm_ctx_alignment() - 1);
                len = ALIGN(len, crypto_tfm_ctx_alignment());
        }

        len += sizeof(struct aead_request) + crypto_aead_reqsize(aead);
        len = ALIGN(len, __alignof__(struct scatterlist));

        len += sizeof(struct scatterlist) * nfrags;

        return kmalloc(len, GFP_ATOMIC);
}

static inline void *esp_tmp_extra(void *tmp)
{
        return PTR_ALIGN(tmp, __alignof__(struct esp_output_extra));
}

static inline u8 *esp_tmp_iv(struct crypto_aead *aead, void *tmp, int seqhilen)
{
        return crypto_aead_ivsize(aead) ?
               PTR_ALIGN((u8 *)tmp + seqhilen,
                         crypto_aead_alignmask(aead) + 1) : tmp + seqhilen;
}

static inline struct aead_request *esp_tmp_req(struct crypto_aead *aead, u8 *iv)
{
        struct aead_request *req;

        req = (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead),
                                crypto_tfm_ctx_alignment());
        aead_request_set_tfm(req, aead);
        return req;
}

static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead,
                                             struct aead_request *req)
{
        return (void *)ALIGN((unsigned long)(req + 1) +
                             crypto_aead_reqsize(aead),
                             __alignof__(struct scatterlist));
}

static void esp_ssg_unref(struct xfrm_state *x, void *tmp, struct sk_buff *skb, bool already_unref)
{
        struct crypto_aead *aead = x->data;
        int extralen = 0;
        u8 *iv;
        struct aead_request *req;
        struct scatterlist *sg;

        if (x->props.flags & XFRM_STATE_ESN)
                extralen += sizeof(struct esp_output_extra);

        iv = esp_tmp_iv(aead, tmp, extralen);
        req = esp_tmp_req(aead, iv);

        /* Unref skb_frag_pages in the src scatterlist if necessary.
         * Skip the first sg which comes from skb->data.
         */
        if (already_unref || req->src != req->dst) {
                struct scatterlist *src = already_unref ? esp_req_sg(aead, req) : req->src;

                for (sg = sg_next(src); sg; sg = sg_next(sg))
                        skb_page_unref(page_to_netmem(sg_page(sg)),
                                       skb->pp_recycle);
        }
}

#ifdef CONFIG_INET6_ESPINTCP
static struct sock *esp6_find_tcp_sk(struct xfrm_state *x)
{
        struct xfrm_encap_tmpl *encap = x->encap;
        struct net *net = xs_net(x);
        __be16 sport, dport;
        struct sock *sk;

        spin_lock_bh(&x->lock);
        sport = encap->encap_sport;
        dport = encap->encap_dport;
        spin_unlock_bh(&x->lock);

        sk = __inet6_lookup_established(net, &x->id.daddr.in6, dport,
                                        &x->props.saddr.in6, ntohs(sport), 0, 0);
        if (!sk)
                return ERR_PTR(-ENOENT);

        if (!tcp_is_ulp_esp(sk)) {
                sock_put(sk);
                return ERR_PTR(-EINVAL);
        }

        return sk;
}

static int esp_output_tcp_finish(struct xfrm_state *x, struct sk_buff *skb)
{
        struct sock *sk;
        int err;

        rcu_read_lock();

        sk = esp6_find_tcp_sk(x);
        err = PTR_ERR_OR_ZERO(sk);
        if (err) {
                kfree_skb(skb);
                goto out;
        }

        bh_lock_sock(sk);
        if (sock_owned_by_user(sk))
                err = espintcp_queue_out(sk, skb);
        else
                err = espintcp_push_skb(sk, skb);
        bh_unlock_sock(sk);

        sock_put(sk);

out:
        rcu_read_unlock();
        return err;
}

static int esp_output_tcp_encap_cb(struct net *net, struct sock *sk,
                                   struct sk_buff *skb)
{
        struct dst_entry *dst = skb_dst(skb);
        struct xfrm_state *x = dst->xfrm;

        return esp_output_tcp_finish(x, skb);
}

static int esp_output_tail_tcp(struct xfrm_state *x, struct sk_buff *skb)
{
        int err;

        local_bh_disable();
        err = xfrm_trans_queue_net(xs_net(x), skb, esp_output_tcp_encap_cb);
        local_bh_enable();

        /* EINPROGRESS just happens to do the right thing.  It
         * actually means that the skb has been consumed and
         * isn't coming back.
         */
        return err ?: -EINPROGRESS;
}
#else
static int esp_output_tail_tcp(struct xfrm_state *x, struct sk_buff *skb)
{
        WARN_ON(1);
        return -EOPNOTSUPP;
}
#endif

static void esp_output_encap_csum(struct sk_buff *skb)
{
        /* UDP encap with IPv6 requires a valid checksum */
        if (*skb_mac_header(skb) == IPPROTO_UDP) {
                struct udphdr *uh = udp_hdr(skb);
                struct ipv6hdr *ip6h = ipv6_hdr(skb);
                int len = ntohs(uh->len);
                unsigned int offset = skb_transport_offset(skb);
                __wsum csum = skb_checksum(skb, offset, skb->len - offset, 0);

                uh->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
                                            len, IPPROTO_UDP, csum);
                if (uh->check == 0)
                        uh->check = CSUM_MANGLED_0;
        }
}

static void esp_output_done(void *data, int err)
{
        struct sk_buff *skb = data;
        struct xfrm_offload *xo = xfrm_offload(skb);
        void *tmp;
        struct xfrm_state *x;

        if (xo && (xo->flags & XFRM_DEV_RESUME)) {
                struct sec_path *sp = skb_sec_path(skb);

                x = sp->xvec[sp->len - 1];
        } else {
                x = skb_dst(skb)->xfrm;
        }

        tmp = ESP_SKB_CB(skb)->tmp;
        esp_ssg_unref(x, tmp, skb, false);
        kfree(tmp);

        esp_output_encap_csum(skb);

        if (xo && (xo->flags & XFRM_DEV_RESUME)) {
                if (err) {
                        XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
                        kfree_skb(skb);
                        return;
                }

                skb_push(skb, skb->data - skb_mac_header(skb));
                secpath_reset(skb);
                xfrm_dev_resume(skb);
        } else {
                if (!err &&
                    x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP) {
                        err = esp_output_tail_tcp(x, skb);
                        if (err != -EINPROGRESS)
                                kfree_skb(skb);
                } else {
                        xfrm_output_resume(skb_to_full_sk(skb), skb, err);
                }
        }
}

/* Move ESP header back into place. */
static void esp_restore_header(struct sk_buff *skb, unsigned int offset)
{
        struct ip_esp_hdr *esph = (void *)(skb->data + offset);
        void *tmp = ESP_SKB_CB(skb)->tmp;
        __be32 *seqhi = esp_tmp_extra(tmp);

        esph->seq_no = esph->spi;
        esph->spi = *seqhi;
}

static void esp_output_restore_header(struct sk_buff *skb)
{
        void *tmp = ESP_SKB_CB(skb)->tmp;
        struct esp_output_extra *extra = esp_tmp_extra(tmp);

        esp_restore_header(skb, skb_transport_offset(skb) + extra->esphoff -
                                sizeof(__be32));
}

static struct ip_esp_hdr *esp_output_set_esn(struct sk_buff *skb,
                                             struct xfrm_state *x,
                                             struct ip_esp_hdr *esph,
                                             struct esp_output_extra *extra)
{
        /* For ESN we move the header forward by 4 bytes to
         * accommodate the high bits.  We will move it back after
         * encryption.
         */
        if ((x->props.flags & XFRM_STATE_ESN)) {
                __u32 seqhi;
                struct xfrm_offload *xo = xfrm_offload(skb);

                if (xo)
                        seqhi = xo->seq.hi;
                else
                        seqhi = XFRM_SKB_CB(skb)->seq.output.hi;

                extra->esphoff = (unsigned char *)esph -
                                 skb_transport_header(skb);
                esph = (struct ip_esp_hdr *)((unsigned char *)esph - 4);
                extra->seqhi = esph->spi;
                esph->seq_no = htonl(seqhi);
        }

        esph->spi = x->id.spi;

        return esph;
}

static void esp_output_done_esn(void *data, int err)
{
        struct sk_buff *skb = data;

        esp_output_restore_header(skb);
        esp_output_done(data, err);
}

static struct ip_esp_hdr *esp6_output_udp_encap(struct sk_buff *skb,
                                               int encap_type,
                                               struct esp_info *esp,
                                               __be16 sport,
                                               __be16 dport)
{
        struct udphdr *uh;
        unsigned int len;

        len = skb->len + esp->tailen - skb_transport_offset(skb);
        if (len > U16_MAX)
                return ERR_PTR(-EMSGSIZE);

        uh = (struct udphdr *)esp->esph;
        uh->source = sport;
        uh->dest = dport;
        uh->len = htons(len);
        uh->check = 0;

        *skb_mac_header(skb) = IPPROTO_UDP;

        return (struct ip_esp_hdr *)(uh + 1);
}

#ifdef CONFIG_INET6_ESPINTCP
static struct ip_esp_hdr *esp6_output_tcp_encap(struct xfrm_state *x,
                                                struct sk_buff *skb,
                                                struct esp_info *esp)
{
        __be16 *lenp = (void *)esp->esph;
        struct ip_esp_hdr *esph;
        unsigned int len;
        struct sock *sk;

        len = skb->len + esp->tailen - skb_transport_offset(skb);
        if (len > IP_MAX_MTU)
                return ERR_PTR(-EMSGSIZE);

        rcu_read_lock();
        sk = esp6_find_tcp_sk(x);
        rcu_read_unlock();

        if (IS_ERR(sk))
                return ERR_CAST(sk);

        sock_put(sk);

        *lenp = htons(len);
        esph = (struct ip_esp_hdr *)(lenp + 1);

        return esph;
}
#else
static struct ip_esp_hdr *esp6_output_tcp_encap(struct xfrm_state *x,
                                                struct sk_buff *skb,
                                                struct esp_info *esp)
{
        return ERR_PTR(-EOPNOTSUPP);
}
#endif

static int esp6_output_encap(struct xfrm_state *x, struct sk_buff *skb,
                            struct esp_info *esp)
{
        struct xfrm_encap_tmpl *encap = x->encap;
        struct ip_esp_hdr *esph;
        __be16 sport, dport;
        int encap_type;

        spin_lock_bh(&x->lock);
        sport = encap->encap_sport;
        dport = encap->encap_dport;
        encap_type = encap->encap_type;
        spin_unlock_bh(&x->lock);

        switch (encap_type) {
        default:
        case UDP_ENCAP_ESPINUDP:
                esph = esp6_output_udp_encap(skb, encap_type, esp, sport, dport);
                break;
        case TCP_ENCAP_ESPINTCP:
                esph = esp6_output_tcp_encap(x, skb, esp);
                break;
        }

        if (IS_ERR(esph))
                return PTR_ERR(esph);

        esp->esph = esph;

        return 0;
}

int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
{
        u8 *tail;
        int nfrags;
        int esph_offset;
        struct page *page;
        struct sk_buff *trailer;
        int tailen = esp->tailen;

        if (x->encap) {
                int err = esp6_output_encap(x, skb, esp);

                if (err < 0)
                        return err;
        }

        if (ALIGN(skb->data_len + tailen, L1_CACHE_BYTES) >
            PAGE_SIZE)
                goto cow;

        if (!skb_cloned(skb)) {
                if (tailen <= skb_tailroom(skb)) {
                        nfrags = 1;
                        trailer = skb;
                        tail = skb_tail_pointer(trailer);

                        goto skip_cow;
                } else if ((skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)
                           && !skb_has_frag_list(skb)) {
                        int allocsize;
                        struct sock *sk = skb->sk;
                        struct page_frag *pfrag = &x->xfrag;

                        esp->inplace = false;

                        allocsize = ALIGN(tailen, L1_CACHE_BYTES);

                        spin_lock_bh(&x->lock);

                        if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
                                spin_unlock_bh(&x->lock);
                                goto cow;
                        }

                        page = pfrag->page;
                        get_page(page);

                        tail = page_address(page) + pfrag->offset;

                        esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);

                        nfrags = skb_shinfo(skb)->nr_frags;

                        __skb_fill_page_desc(skb, nfrags, page, pfrag->offset,
                                             tailen);
                        skb_shinfo(skb)->nr_frags = ++nfrags;

                        pfrag->offset = pfrag->offset + allocsize;

                        spin_unlock_bh(&x->lock);

                        nfrags++;

                        skb->len += tailen;
                        skb->data_len += tailen;
                        skb->truesize += tailen;
                        if (sk && sk_fullsock(sk))
                                refcount_add(tailen, &sk->sk_wmem_alloc);

                        goto out;
                }
        }

cow:
        esph_offset = (unsigned char *)esp->esph - skb_transport_header(skb);

        nfrags = skb_cow_data(skb, tailen, &trailer);
        if (nfrags < 0)
                goto out;
        tail = skb_tail_pointer(trailer);
        esp->esph = (struct ip_esp_hdr *)(skb_transport_header(skb) + esph_offset);

skip_cow:
        esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);
        pskb_put(skb, trailer, tailen);

out:
        return nfrags;
}
EXPORT_SYMBOL_GPL(esp6_output_head);

int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
{
        u8 *iv;
        int alen;
        void *tmp;
        int ivlen;
        int assoclen;
        int extralen;
        struct page *page;
        struct ip_esp_hdr *esph;
        struct aead_request *req;
        struct crypto_aead *aead;
        struct scatterlist *sg, *dsg;
        struct esp_output_extra *extra;
        int err = -ENOMEM;

        assoclen = sizeof(struct ip_esp_hdr);
        extralen = 0;

        if (x->props.flags & XFRM_STATE_ESN) {
                extralen += sizeof(*extra);
                assoclen += sizeof(__be32);
        }

        aead = x->data;
        alen = crypto_aead_authsize(aead);
        ivlen = crypto_aead_ivsize(aead);

        tmp = esp_alloc_tmp(aead, esp->nfrags + 2, extralen);
        if (!tmp)
                goto error;

        extra = esp_tmp_extra(tmp);
        iv = esp_tmp_iv(aead, tmp, extralen);
        req = esp_tmp_req(aead, iv);
        sg = esp_req_sg(aead, req);

        if (esp->inplace)
                dsg = sg;
        else
                dsg = &sg[esp->nfrags];

        esph = esp_output_set_esn(skb, x, esp->esph, extra);
        esp->esph = esph;

        sg_init_table(sg, esp->nfrags);
        err = skb_to_sgvec(skb, sg,
                           (unsigned char *)esph - skb->data,
                           assoclen + ivlen + esp->clen + alen);
        if (unlikely(err < 0))
                goto error_free;

        if (!esp->inplace) {
                int allocsize;
                struct page_frag *pfrag = &x->xfrag;

                allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES);

                spin_lock_bh(&x->lock);
                if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
                        spin_unlock_bh(&x->lock);
                        goto error_free;
                }

                skb_shinfo(skb)->nr_frags = 1;

                page = pfrag->page;
                get_page(page);
                /* replace page frags in skb with new page */
                __skb_fill_page_desc(skb, 0, page, pfrag->offset, skb->data_len);
                pfrag->offset = pfrag->offset + allocsize;
                spin_unlock_bh(&x->lock);

                sg_init_table(dsg, skb_shinfo(skb)->nr_frags + 1);
                err = skb_to_sgvec(skb, dsg,
                                   (unsigned char *)esph - skb->data,
                                   assoclen + ivlen + esp->clen + alen);
                if (unlikely(err < 0)) {
                        esp_ssg_unref(x, tmp, skb, true);
                        goto error_free;
                }
        }

        if ((x->props.flags & XFRM_STATE_ESN))
                aead_request_set_callback(req, 0, esp_output_done_esn, skb);
        else
                aead_request_set_callback(req, 0, esp_output_done, skb);

        aead_request_set_crypt(req, sg, dsg, ivlen + esp->clen, iv);
        aead_request_set_ad(req, assoclen);

        memset(iv, 0, ivlen);
        memcpy(iv + ivlen - min(ivlen, 8), (u8 *)&esp->seqno + 8 - min(ivlen, 8),
               min(ivlen, 8));

        ESP_SKB_CB(skb)->tmp = tmp;
        err = crypto_aead_encrypt(req);

        switch (err) {
        case -EINPROGRESS:
                goto error;

        case -ENOSPC:
                err = NET_XMIT_DROP;
                break;

        case 0:
                if ((x->props.flags & XFRM_STATE_ESN))
                        esp_output_restore_header(skb);
                esp_output_encap_csum(skb);
        }

        if (sg != dsg)
                esp_ssg_unref(x, tmp, skb, false);

        if (!err && x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP)
                err = esp_output_tail_tcp(x, skb);

error_free:
        kfree(tmp);
error:
        return err;
}
EXPORT_SYMBOL_GPL(esp6_output_tail);

static int esp6_output(struct xfrm_state *x, struct sk_buff *skb)
{
        int alen;
        int blksize;
        struct ip_esp_hdr *esph;
        struct crypto_aead *aead;
        struct esp_info esp;

        esp.inplace = true;

        esp.proto = *skb_mac_header(skb);
        *skb_mac_header(skb) = IPPROTO_ESP;

        /* skb is pure payload to encrypt */

        aead = x->data;
        alen = crypto_aead_authsize(aead);

        esp.tfclen = 0;
        if (x->tfcpad) {
                struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb);
                u32 padto;

                padto = min(x->tfcpad, xfrm_state_mtu(x, dst->child_mtu_cached));
                if (skb->len < padto)
                        esp.tfclen = padto - skb->len;
        }
        blksize = ALIGN(crypto_aead_blocksize(aead), 4);
        esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
        esp.plen = esp.clen - skb->len - esp.tfclen;
        esp.tailen = esp.tfclen + esp.plen + alen;

        esp.esph = ip_esp_hdr(skb);

        esp.nfrags = esp6_output_head(x, skb, &esp);
        if (esp.nfrags < 0)
                return esp.nfrags;

        esph = esp.esph;
        esph->spi = x->id.spi;

        esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
        esp.seqno = cpu_to_be64(XFRM_SKB_CB(skb)->seq.output.low +
                            ((u64)XFRM_SKB_CB(skb)->seq.output.hi << 32));

        skb_push(skb, -skb_network_offset(skb));

        return esp6_output_tail(x, skb, &esp);
}

static inline int esp_remove_trailer(struct sk_buff *skb)
{
        struct xfrm_state *x = xfrm_input_state(skb);
        struct crypto_aead *aead = x->data;
        int alen, hlen, elen;
        int padlen, trimlen;
        __wsum csumdiff;
        u8 nexthdr[2];
        int ret;

        alen = crypto_aead_authsize(aead);
        hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
        elen = skb->len - hlen;

        ret = skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2);
        BUG_ON(ret);

        ret = -EINVAL;
        padlen = nexthdr[0];
        if (padlen + 2 + alen >= elen) {
                net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",
                                    padlen + 2, elen - alen);
                goto out;
        }

        trimlen = alen + padlen + 2;
        if (skb->ip_summed == CHECKSUM_COMPLETE) {
                csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);
                skb->csum = csum_block_sub(skb->csum, csumdiff,
                                           skb->len - trimlen);
        }
        ret = pskb_trim(skb, skb->len - trimlen);
        if (unlikely(ret))
                return ret;

        ret = nexthdr[1];

out:
        return ret;
}

int esp6_input_done2(struct sk_buff *skb, int err)
{
        struct xfrm_state *x = xfrm_input_state(skb);
        struct xfrm_offload *xo = xfrm_offload(skb);
        struct crypto_aead *aead = x->data;
        int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
        int hdr_len = skb_network_header_len(skb);

        if (!xo || !(xo->flags & CRYPTO_DONE))
                kfree(ESP_SKB_CB(skb)->tmp);

        if (unlikely(err))
                goto out;

        err = esp_remove_trailer(skb);
        if (unlikely(err < 0))
                goto out;

        if (x->encap) {
                const struct ipv6hdr *ip6h = ipv6_hdr(skb);
                int offset = skb_network_offset(skb) + sizeof(*ip6h);
                struct xfrm_encap_tmpl *encap = x->encap;
                u8 nexthdr = ip6h->nexthdr;
                __be16 frag_off, source;
                struct udphdr *uh;
                struct tcphdr *th;

                offset = ipv6_skip_exthdr(skb, offset, &nexthdr, &frag_off);
                if (offset == -1) {
                        err = -EINVAL;
                        goto out;
                }

                uh = (void *)(skb->data + offset);
                th = (void *)(skb->data + offset);
                hdr_len += offset;

                switch (x->encap->encap_type) {
                case TCP_ENCAP_ESPINTCP:
                        source = th->source;
                        break;
                case UDP_ENCAP_ESPINUDP:
                        source = uh->source;
                        break;
                default:
                        WARN_ON_ONCE(1);
                        err = -EINVAL;
                        goto out;
                }

                /*
                 * 1) if the NAT-T peer's IP or port changed then
                 *    advertise the change to the keying daemon.
                 *    This is an inbound SA, so just compare
                 *    SRC ports.
                 */
                if (!ipv6_addr_equal(&ip6h->saddr, &x->props.saddr.in6) ||
                    source != encap->encap_sport) {
                        xfrm_address_t ipaddr;

                        memcpy(&ipaddr.a6, &ip6h->saddr.s6_addr, sizeof(ipaddr.a6));
                        km_new_mapping(x, &ipaddr, source);

                        /* XXX: perhaps add an extra
                         * policy check here, to see
                         * if we should allow or
                         * reject a packet from a
                         * different source
                         * address/port.
                         */
                }

                /*
                 * 2) ignore UDP/TCP checksums in case
                 *    of NAT-T in Transport Mode, or
                 *    perform other post-processing fixes
                 *    as per draft-ietf-ipsec-udp-encaps-06,
                 *    section 3.1.2
                 */
                if (x->props.mode == XFRM_MODE_TRANSPORT)
                        skb->ip_summed = CHECKSUM_UNNECESSARY;
        }

        skb_postpull_rcsum(skb, skb_network_header(skb),
                           skb_network_header_len(skb));
        skb_pull_rcsum(skb, hlen);
        if (x->props.mode == XFRM_MODE_TUNNEL ||
            x->props.mode == XFRM_MODE_IPTFS)
                skb_reset_transport_header(skb);
        else
                skb_set_transport_header(skb, -hdr_len);

        /* RFC4303: Drop dummy packets without any error */
        if (err == IPPROTO_NONE)
                err = -EINVAL;

out:
        return err;
}
EXPORT_SYMBOL_GPL(esp6_input_done2);

static void esp_input_done(void *data, int err)
{
        struct sk_buff *skb = data;

        xfrm_input_resume(skb, esp6_input_done2(skb, err));
}

static void esp_input_restore_header(struct sk_buff *skb)
{
        esp_restore_header(skb, 0);
        __skb_pull(skb, 4);
}

static void esp_input_set_header(struct sk_buff *skb, __be32 *seqhi)
{
        struct xfrm_state *x = xfrm_input_state(skb);

        /* For ESN we move the header forward by 4 bytes to
         * accommodate the high bits.  We will move it back after
         * decryption.
         */
        if ((x->props.flags & XFRM_STATE_ESN)) {
                struct ip_esp_hdr *esph = skb_push(skb, 4);

                *seqhi = esph->spi;
                esph->spi = esph->seq_no;
                esph->seq_no = XFRM_SKB_CB(skb)->seq.input.hi;
        }
}

static void esp_input_done_esn(void *data, int err)
{
        struct sk_buff *skb = data;

        esp_input_restore_header(skb);
        esp_input_done(data, err);
}

static int esp6_input(struct xfrm_state *x, struct sk_buff *skb)
{
        struct crypto_aead *aead = x->data;
        struct aead_request *req;
        struct sk_buff *trailer;
        int ivlen = crypto_aead_ivsize(aead);
        int elen = skb->len - sizeof(struct ip_esp_hdr) - ivlen;
        int nfrags;
        int assoclen;
        int seqhilen;
        int ret = 0;
        void *tmp;
        __be32 *seqhi;
        u8 *iv;
        struct scatterlist *sg;

        if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + ivlen)) {
                ret = -EINVAL;
                goto out;
        }

        if (elen <= 0) {
                ret = -EINVAL;
                goto out;
        }

        assoclen = sizeof(struct ip_esp_hdr);
        seqhilen = 0;

        if (x->props.flags & XFRM_STATE_ESN) {
                seqhilen += sizeof(__be32);
                assoclen += seqhilen;
        }

        if (!skb_cloned(skb)) {
                if (!skb_is_nonlinear(skb)) {
                        nfrags = 1;

                        goto skip_cow;
                } else if (!skb_has_frag_list(skb) &&
                           !skb_has_shared_frag(skb)) {
                        nfrags = skb_shinfo(skb)->nr_frags;
                        nfrags++;

                        goto skip_cow;
                }
        }

        nfrags = skb_cow_data(skb, 0, &trailer);
        if (nfrags < 0) {
                ret = -EINVAL;
                goto out;
        }

skip_cow:
        ret = -ENOMEM;
        tmp = esp_alloc_tmp(aead, nfrags, seqhilen);
        if (!tmp)
                goto out;

        ESP_SKB_CB(skb)->tmp = tmp;
        seqhi = esp_tmp_extra(tmp);
        iv = esp_tmp_iv(aead, tmp, seqhilen);
        req = esp_tmp_req(aead, iv);
        sg = esp_req_sg(aead, req);

        esp_input_set_header(skb, seqhi);

        sg_init_table(sg, nfrags);
        ret = skb_to_sgvec(skb, sg, 0, skb->len);
        if (unlikely(ret < 0)) {
                kfree(tmp);
                goto out;
        }

        skb->ip_summed = CHECKSUM_NONE;

        if ((x->props.flags & XFRM_STATE_ESN))
                aead_request_set_callback(req, 0, esp_input_done_esn, skb);
        else
                aead_request_set_callback(req, 0, esp_input_done, skb);

        aead_request_set_crypt(req, sg, sg, elen + ivlen, iv);
        aead_request_set_ad(req, assoclen);

        ret = crypto_aead_decrypt(req);
        if (ret == -EINPROGRESS)
                goto out;

        if ((x->props.flags & XFRM_STATE_ESN))
                esp_input_restore_header(skb);

        ret = esp6_input_done2(skb, ret);

out:
        return ret;
}

static int esp6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                    u8 type, u8 code, int offset, __be32 info)
{
        struct net *net = dev_net(skb->dev);
        const struct ipv6hdr *iph = (const struct ipv6hdr *)skb->data;
        struct ip_esp_hdr *esph = (struct ip_esp_hdr *)(skb->data + offset);
        struct xfrm_state *x;

        if (type != ICMPV6_PKT_TOOBIG &&
            type != NDISC_REDIRECT)
                return 0;

        x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr,
                              esph->spi, IPPROTO_ESP, AF_INET6);
        if (!x)
                return 0;

        if (type == NDISC_REDIRECT)
                ip6_redirect(skb, net, skb->dev->ifindex, 0,
                             sock_net_uid(net, NULL));
        else
                ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL));
        xfrm_state_put(x);

        return 0;
}

static void esp6_destroy(struct xfrm_state *x)
{
        struct crypto_aead *aead = x->data;

        if (!aead)
                return;

        crypto_free_aead(aead);
}

static int esp_init_aead(struct xfrm_state *x, struct netlink_ext_ack *extack)
{
        char aead_name[CRYPTO_MAX_ALG_NAME];
        struct crypto_aead *aead;
        int err;

        if (snprintf(aead_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
                     x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME) {
                NL_SET_ERR_MSG(extack, "Algorithm name is too long");
                return -ENAMETOOLONG;
        }

        aead = crypto_alloc_aead(aead_name, 0, 0);
        err = PTR_ERR(aead);
        if (IS_ERR(aead))
                goto error;

        x->data = aead;

        err = crypto_aead_setkey(aead, x->aead->alg_key,
                                 (x->aead->alg_key_len + 7) / 8);
        if (err)
                goto error;

        err = crypto_aead_setauthsize(aead, x->aead->alg_icv_len / 8);
        if (err)
                goto error;

        return 0;

error:
        NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
        return err;
}

static int esp_init_authenc(struct xfrm_state *x,
                            struct netlink_ext_ack *extack)
{
        struct crypto_aead *aead;
        struct crypto_authenc_key_param *param;
        struct rtattr *rta;
        char *key;
        char *p;
        char authenc_name[CRYPTO_MAX_ALG_NAME];
        unsigned int keylen;
        int err;

        err = -ENAMETOOLONG;

        if ((x->props.flags & XFRM_STATE_ESN)) {
                if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME,
                             "%s%sauthencesn(%s,%s)%s",
                             x->geniv ?: "", x->geniv ? "(" : "",
                             x->aalg ? x->aalg->alg_name : "digest_null",
                             x->ealg->alg_name,
                             x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) {
                        NL_SET_ERR_MSG(extack, "Algorithm name is too long");
                        goto error;
                }
        } else {
                if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME,
                             "%s%sauthenc(%s,%s)%s",
                             x->geniv ?: "", x->geniv ? "(" : "",
                             x->aalg ? x->aalg->alg_name : "digest_null",
                             x->ealg->alg_name,
                             x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) {
                        NL_SET_ERR_MSG(extack, "Algorithm name is too long");
                        goto error;
                }
        }

        aead = crypto_alloc_aead(authenc_name, 0, 0);
        err = PTR_ERR(aead);
        if (IS_ERR(aead)) {
                NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
                goto error;
        }

        x->data = aead;

        keylen = (x->aalg ? (x->aalg->alg_key_len + 7) / 8 : 0) +
                 (x->ealg->alg_key_len + 7) / 8 + RTA_SPACE(sizeof(*param));
        err = -ENOMEM;
        key = kmalloc(keylen, GFP_KERNEL);
        if (!key)
                goto error;

        p = key;
        rta = (void *)p;
        rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
        rta->rta_len = RTA_LENGTH(sizeof(*param));
        param = RTA_DATA(rta);
        p += RTA_SPACE(sizeof(*param));

        if (x->aalg) {
                struct xfrm_algo_desc *aalg_desc;

                memcpy(p, x->aalg->alg_key, (x->aalg->alg_key_len + 7) / 8);
                p += (x->aalg->alg_key_len + 7) / 8;

                aalg_desc = xfrm_aalg_get_byname(x->aalg->alg_name, 0);
                BUG_ON(!aalg_desc);

                err = -EINVAL;
                if (aalg_desc->uinfo.auth.icv_fullbits / 8 !=
                    crypto_aead_authsize(aead)) {
                        NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
                        goto free_key;
                }

                err = crypto_aead_setauthsize(
                        aead, x->aalg->alg_trunc_len / 8);
                if (err) {
                        NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
                        goto free_key;
                }
        }

        param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8);
        memcpy(p, x->ealg->alg_key, (x->ealg->alg_key_len + 7) / 8);

        err = crypto_aead_setkey(aead, key, keylen);

free_key:
        kfree(key);

error:
        return err;
}

static int esp6_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
{
        struct crypto_aead *aead;
        u32 align;
        int err;

        x->data = NULL;

        if (x->aead) {
                err = esp_init_aead(x, extack);
        } else if (x->ealg) {
                err = esp_init_authenc(x, extack);
        } else {
                NL_SET_ERR_MSG(extack, "ESP: AEAD or CRYPT must be provided");
                err = -EINVAL;
        }

        if (err)
                goto error;

        aead = x->data;

        x->props.header_len = sizeof(struct ip_esp_hdr) +
                              crypto_aead_ivsize(aead);
        switch (x->props.mode) {
        case XFRM_MODE_BEET:
                if (x->sel.family != AF_INET6)
                        x->props.header_len += IPV4_BEET_PHMAXLEN +
                                               (sizeof(struct ipv6hdr) - sizeof(struct iphdr));
                break;
        default:
        case XFRM_MODE_TRANSPORT:
                break;
        case XFRM_MODE_TUNNEL:
                x->props.header_len += sizeof(struct ipv6hdr);
                break;
        }

        if (x->encap) {
                struct xfrm_encap_tmpl *encap = x->encap;

                switch (encap->encap_type) {
                default:
                        NL_SET_ERR_MSG(extack, "Unsupported encapsulation type for ESP");
                        err = -EINVAL;
                        goto error;
                case UDP_ENCAP_ESPINUDP:
                        x->props.header_len += sizeof(struct udphdr);
                        break;
#ifdef CONFIG_INET6_ESPINTCP
                case TCP_ENCAP_ESPINTCP:
                        /* only the length field, TCP encap is done by
                         * the socket
                         */
                        x->props.header_len += 2;
                        break;
#endif
                }
        }

        align = ALIGN(crypto_aead_blocksize(aead), 4);
        x->props.trailer_len = align + 1 + crypto_aead_authsize(aead);

error:
        return err;
}

static int esp6_rcv_cb(struct sk_buff *skb, int err)
{
        return 0;
}

static const struct xfrm_type esp6_type = {
        .owner                = THIS_MODULE,
        .proto                = IPPROTO_ESP,
        .flags                = XFRM_TYPE_REPLAY_PROT,
        .init_state        = esp6_init_state,
        .destructor        = esp6_destroy,
        .input                = esp6_input,
        .output                = esp6_output,
};

static struct xfrm6_protocol esp6_protocol = {
        .handler        =        xfrm6_rcv,
        .input_handler        =        xfrm_input,
        .cb_handler        =        esp6_rcv_cb,
        .err_handler        =        esp6_err,
        .priority        =        0,
};

static int __init esp6_init(void)
{
        if (xfrm_register_type(&esp6_type, AF_INET6) < 0) {
                pr_info("%s: can't add xfrm type\n", __func__);
                return -EAGAIN;
        }
        if (xfrm6_protocol_register(&esp6_protocol, IPPROTO_ESP) < 0) {
                pr_info("%s: can't add protocol\n", __func__);
                xfrm_unregister_type(&esp6_type, AF_INET6);
                return -EAGAIN;
        }

        return 0;
}

static void __exit esp6_fini(void)
{
        if (xfrm6_protocol_deregister(&esp6_protocol, IPPROTO_ESP) < 0)
                pr_info("%s: can't remove protocol\n", __func__);
        xfrm_unregister_type(&esp6_type, AF_INET6);
}

module_init(esp6_init);
module_exit(esp6_fini);

MODULE_DESCRIPTION("IPv6 ESP transformation helpers");
MODULE_LICENSE("GPL");
MODULE_ALIAS_XFRM_TYPE(AF_INET6, XFRM_PROTO_ESP);










































































































    5 




























































































































































































































































































































































































































































































































































































































































































































































 1739 




 1739 
 1744 








 1743 






















































































































































    5 
    5 







    5 






























































 1736 






 1739 

















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  Copyright (C) 2008 Red Hat, Inc., Eric Paris <eparis@redhat.com>
 */

/*
 * fsnotify inode mark locking/lifetime/and refcnting
 *
 * REFCNT:
 * The group->recnt and mark->refcnt tell how many "things" in the kernel
 * currently are referencing the objects. Both kind of objects typically will
 * live inside the kernel with a refcnt of 2, one for its creation and one for
 * the reference a group and a mark hold to each other.
 * If you are holding the appropriate locks, you can take a reference and the
 * object itself is guaranteed to survive until the reference is dropped.
 *
 * LOCKING:
 * There are 3 locks involved with fsnotify inode marks and they MUST be taken
 * in order as follows:
 *
 * group->mark_mutex
 * mark->lock
 * mark->connector->lock
 *
 * group->mark_mutex protects the marks_list anchored inside a given group and
 * each mark is hooked via the g_list.  It also protects the groups private
 * data (i.e group limits).

 * mark->lock protects the marks attributes like its masks and flags.
 * Furthermore it protects the access to a reference of the group that the mark
 * is assigned to as well as the access to a reference of the inode/vfsmount
 * that is being watched by the mark.
 *
 * mark->connector->lock protects the list of marks anchored inside an
 * inode / vfsmount and each mark is hooked via the i_list.
 *
 * A list of notification marks relating to inode / mnt is contained in
 * fsnotify_mark_connector. That structure is alive as long as there are any
 * marks in the list and is also protected by fsnotify_mark_srcu. A mark gets
 * detached from fsnotify_mark_connector when last reference to the mark is
 * dropped.  Thus having mark reference is enough to protect mark->connector
 * pointer and to make sure fsnotify_mark_connector cannot disappear. Also
 * because we remove mark from g_list before dropping mark reference associated
 * with that, any mark found through g_list is guaranteed to have
 * mark->connector set until we drop group->mark_mutex.
 *
 * LIFETIME:
 * Inode marks survive between when they are added to an inode and when their
 * refcnt==0. Marks are also protected by fsnotify_mark_srcu.
 *
 * The inode mark can be cleared for a number of different reasons including:
 * - The inode is unlinked for the last time.  (fsnotify_inode_remove)
 * - The inode is being evicted from cache. (fsnotify_inode_delete)
 * - The fs the inode is on is unmounted.  (fsnotify_inode_delete/fsnotify_unmount_inodes)
 * - Something explicitly requests that it be removed.  (fsnotify_destroy_mark)
 * - The fsnotify_group associated with the mark is going away and all such marks
 *   need to be cleaned up. (fsnotify_clear_marks_by_group)
 *
 * This has the very interesting property of being able to run concurrently with
 * any (or all) other directions.
 */

#include <linux/fs.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/srcu.h>
#include <linux/ratelimit.h>

#include <linux/atomic.h>

#include <linux/fsnotify_backend.h>
#include "fsnotify.h"

#define FSNOTIFY_REAPER_DELAY        (1)        /* 1 jiffy */

struct srcu_struct fsnotify_mark_srcu;
static struct kmem_cache *fsnotify_mark_connector_cachep;
static struct kmem_cache *fsnotify_inode_mark_connector_cachep;

static DEFINE_SPINLOCK(destroy_lock);
static LIST_HEAD(destroy_list);
static struct fsnotify_mark_connector *connector_destroy_list;

static void fsnotify_mark_destroy_workfn(struct work_struct *work);
static DECLARE_DELAYED_WORK(reaper_work, fsnotify_mark_destroy_workfn);

static void fsnotify_connector_destroy_workfn(struct work_struct *work);
static DECLARE_WORK(connector_reaper_work, fsnotify_connector_destroy_workfn);

void fsnotify_get_mark(struct fsnotify_mark *mark)
{
        WARN_ON_ONCE(!refcount_read(&mark->refcnt));
        refcount_inc(&mark->refcnt);
}

static fsnotify_connp_t *fsnotify_object_connp(void *obj,
                                enum fsnotify_obj_type obj_type)
{
        switch (obj_type) {
        case FSNOTIFY_OBJ_TYPE_INODE:
                return &((struct inode *)obj)->i_fsnotify_marks;
        case FSNOTIFY_OBJ_TYPE_VFSMOUNT:
                return &real_mount(obj)->mnt_fsnotify_marks;
        case FSNOTIFY_OBJ_TYPE_SB:
                return fsnotify_sb_marks(obj);
        case FSNOTIFY_OBJ_TYPE_MNTNS:
                return &((struct mnt_namespace *)obj)->n_fsnotify_marks;
        default:
                return NULL;
        }
}

static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn)
{
        if (conn->type == FSNOTIFY_OBJ_TYPE_INODE)
                return &fsnotify_conn_inode(conn)->i_fsnotify_mask;
        else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT)
                return &fsnotify_conn_mount(conn)->mnt_fsnotify_mask;
        else if (conn->type == FSNOTIFY_OBJ_TYPE_SB)
                return &fsnotify_conn_sb(conn)->s_fsnotify_mask;
        else if (conn->type == FSNOTIFY_OBJ_TYPE_MNTNS)
                return &fsnotify_conn_mntns(conn)->n_fsnotify_mask;
        return NULL;
}

__u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn)
{
        if (WARN_ON(!fsnotify_valid_obj_type(conn->type)))
                return 0;

        return READ_ONCE(*fsnotify_conn_mask_p(conn));
}

static void fsnotify_get_sb_watched_objects(struct super_block *sb)
{
        atomic_long_inc(fsnotify_sb_watched_objects(sb));
}

static void fsnotify_put_sb_watched_objects(struct super_block *sb)
{
        atomic_long_t *watched_objects = fsnotify_sb_watched_objects(sb);

        /* the superblock can go away after this decrement */
        if (atomic_long_dec_and_test(watched_objects))
                wake_up_var(watched_objects);
}

static void fsnotify_get_inode_ref(struct inode *inode)
{
        ihold(inode);
        fsnotify_get_sb_watched_objects(inode->i_sb);
}

static void fsnotify_put_inode_ref(struct inode *inode)
{
        /* read ->i_sb before the inode can go away */
        struct super_block *sb = inode->i_sb;

        iput(inode);
        fsnotify_put_sb_watched_objects(sb);
}

/*
 * Grab or drop watched objects reference depending on whether the connector
 * is attached and has any marks attached.
 */
static void fsnotify_update_sb_watchers(struct super_block *sb,
                                        struct fsnotify_mark_connector *conn)
{
        struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb);
        bool is_watched = conn->flags & FSNOTIFY_CONN_FLAG_IS_WATCHED;
        struct fsnotify_mark *first_mark = NULL;
        unsigned int highest_prio = 0;

        if (conn->obj)
                first_mark = hlist_entry_safe(conn->list.first,
                                              struct fsnotify_mark, obj_list);
        if (first_mark)
                highest_prio = first_mark->group->priority;
        if (WARN_ON(highest_prio >= __FSNOTIFY_PRIO_NUM))
                highest_prio = 0;

        /*
         * If the highest priority of group watching this object is prio,
         * then watched object has a reference on counters [0..prio].
         * Update priority >= 1 watched objects counters.
         */
        for (unsigned int p = conn->prio + 1; p <= highest_prio; p++)
                atomic_long_inc(&sbinfo->watched_objects[p]);
        for (unsigned int p = conn->prio; p > highest_prio; p--)
                atomic_long_dec(&sbinfo->watched_objects[p]);
        conn->prio = highest_prio;

        /* Update priority >= 0 (a.k.a total) watched objects counter */
        BUILD_BUG_ON(FSNOTIFY_PRIO_NORMAL != 0);
        if (first_mark && !is_watched) {
                conn->flags |= FSNOTIFY_CONN_FLAG_IS_WATCHED;
                fsnotify_get_sb_watched_objects(sb);
        } else if (!first_mark && is_watched) {
                conn->flags &= ~FSNOTIFY_CONN_FLAG_IS_WATCHED;
                fsnotify_put_sb_watched_objects(sb);
        }
}

/*
 * Grab or drop inode reference for the connector if needed.
 *
 * When it's time to drop the reference, we only clear the HAS_IREF flag and
 * return the inode object. fsnotify_drop_object() will be resonsible for doing
 * iput() outside of spinlocks. This happens when last mark that wanted iref is
 * detached.
 */
static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn,
                                          bool want_iref)
{
        bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;
        struct inode *inode = NULL;

        if (conn->type != FSNOTIFY_OBJ_TYPE_INODE ||
            want_iref == has_iref)
                return NULL;

        if (want_iref) {
                /* Pin inode if any mark wants inode refcount held */
                fsnotify_get_inode_ref(fsnotify_conn_inode(conn));
                conn->flags |= FSNOTIFY_CONN_FLAG_HAS_IREF;
        } else {
                /* Unpin inode after detach of last mark that wanted iref */
                inode = fsnotify_conn_inode(conn);
                conn->flags &= ~FSNOTIFY_CONN_FLAG_HAS_IREF;
        }

        return inode;
}

/*
 * Calculate mask of events for a list of marks.
 *
 * Return true if any of the attached marks want to hold an inode reference.
 */
static bool __fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
{
        u32 new_mask = 0;
        bool want_iref = false;
        struct fsnotify_mark *mark;

        assert_spin_locked(&conn->lock);
        /* We can get detached connector here when inode is getting unlinked. */
        if (!fsnotify_valid_obj_type(conn->type))
                return NULL;
        hlist_for_each_entry(mark, &conn->list, obj_list) {
                if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED))
                        continue;
                new_mask |= fsnotify_calc_mask(mark);
                if (conn->type == FSNOTIFY_OBJ_TYPE_INODE &&
                    !(mark->flags & FSNOTIFY_MARK_FLAG_NO_IREF))
                        want_iref = true;
        }
        /*
         * We use WRITE_ONCE() to prevent silly compiler optimizations from
         * confusing readers not holding conn->lock with partial updates.
         */
        WRITE_ONCE(*fsnotify_conn_mask_p(conn), new_mask);

        return want_iref;
}

/*
 * Calculate mask of events for a list of marks after attach/modify mark
 * and get an inode reference for the connector if needed.
 *
 * A concurrent add of evictable mark and detach of non-evictable mark can
 * lead to __fsnotify_recalc_mask() returning false want_iref, but in this
 * case we defer clearing iref to fsnotify_recalc_mask_clear_iref() called
 * from fsnotify_put_mark().
 */
static void fsnotify_recalc_mask_set_iref(struct fsnotify_mark_connector *conn)
{
        bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF;
        bool want_iref = __fsnotify_recalc_mask(conn) || has_iref;

        (void) fsnotify_update_iref(conn, want_iref);
}

/*
 * Calculate mask of events for a list of marks after detach mark
 * and return the inode object if its reference is no longer needed.
 */
static void *fsnotify_recalc_mask_clear_iref(struct fsnotify_mark_connector *conn)
{
        bool want_iref = __fsnotify_recalc_mask(conn);

        return fsnotify_update_iref(conn, want_iref);
}

static bool fsnotify_conn_watches_children(
                                        struct fsnotify_mark_connector *conn)
{
        if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
                return false;

        return fsnotify_inode_watches_children(fsnotify_conn_inode(conn));
}

static void fsnotify_conn_set_children_dentry_flags(
                                        struct fsnotify_mark_connector *conn)
{
        if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
                return;

        fsnotify_set_children_dentry_flags(fsnotify_conn_inode(conn));
}

/*
 * Calculate mask of events for a list of marks. The caller must make sure
 * connector and connector->obj cannot disappear under us.  Callers achieve
 * this by holding a mark->lock or mark->group->mark_mutex for a mark on this
 * list.
 */
void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn)
{
        bool update_children;

        if (!conn)
                return;

        spin_lock(&conn->lock);
        update_children = !fsnotify_conn_watches_children(conn);
        fsnotify_recalc_mask_set_iref(conn);
        update_children &= fsnotify_conn_watches_children(conn);
        spin_unlock(&conn->lock);
        /*
         * Set children's PARENT_WATCHED flags only if parent started watching.
         * When parent stops watching, we clear false positive PARENT_WATCHED
         * flags lazily in __fsnotify_parent().
         */
        if (update_children)
                fsnotify_conn_set_children_dentry_flags(conn);
}

/* Free all connectors queued for freeing once SRCU period ends */
static void fsnotify_connector_destroy_workfn(struct work_struct *work)
{
        struct fsnotify_mark_connector *conn, *free;

        spin_lock(&destroy_lock);
        conn = connector_destroy_list;
        connector_destroy_list = NULL;
        spin_unlock(&destroy_lock);

        synchronize_srcu(&fsnotify_mark_srcu);
        while (conn) {
                free = conn;
                conn = conn->destroy_next;
                kfree(free);
        }
}

static void fsnotify_untrack_connector(struct fsnotify_mark_connector *conn);

static void *fsnotify_detach_connector_from_object(
                                        struct fsnotify_mark_connector *conn,
                                        unsigned int *type)
{
        fsnotify_connp_t *connp = fsnotify_object_connp(conn->obj, conn->type);
        struct super_block *sb = fsnotify_connector_sb(conn);
        struct inode *inode = NULL;

        *type = conn->type;
        if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED)
                return NULL;

        if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) {
                inode = fsnotify_conn_inode(conn);
                inode->i_fsnotify_mask = 0;
                fsnotify_untrack_connector(conn);

                /* Unpin inode when detaching from connector */
                if (!(conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF))
                        inode = NULL;
        } else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) {
                fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0;
        } else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) {
                fsnotify_conn_sb(conn)->s_fsnotify_mask = 0;
        } else if (conn->type == FSNOTIFY_OBJ_TYPE_MNTNS) {
                fsnotify_conn_mntns(conn)->n_fsnotify_mask = 0;
        }

        rcu_assign_pointer(*connp, NULL);
        conn->obj = NULL;
        conn->type = FSNOTIFY_OBJ_TYPE_DETACHED;
        if (sb)
                fsnotify_update_sb_watchers(sb, conn);

        return inode;
}

static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark)
{
        struct fsnotify_group *group = mark->group;

        if (WARN_ON_ONCE(!group))
                return;
        group->ops->free_mark(mark);
        fsnotify_put_group(group);
}

/* Drop object reference originally held by a connector */
static void fsnotify_drop_object(unsigned int type, void *objp)
{
        if (!objp)
                return;
        /* Currently only inode references are passed to be dropped */
        if (WARN_ON_ONCE(type != FSNOTIFY_OBJ_TYPE_INODE))
                return;
        fsnotify_put_inode_ref(objp);
}

void fsnotify_put_mark(struct fsnotify_mark *mark)
{
        struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector);
        void *objp = NULL;
        unsigned int type = FSNOTIFY_OBJ_TYPE_DETACHED;
        bool free_conn = false;

        /* Catch marks that were actually never attached to object */
        if (!conn) {
                if (refcount_dec_and_test(&mark->refcnt))
                        fsnotify_final_mark_destroy(mark);
                return;
        }

        /*
         * We have to be careful so that traversals of obj_list under lock can
         * safely grab mark reference.
         */
        if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock))
                return;

        hlist_del_init_rcu(&mark->obj_list);
        if (hlist_empty(&conn->list)) {
                objp = fsnotify_detach_connector_from_object(conn, &type);
                free_conn = true;
        } else {
                struct super_block *sb = fsnotify_connector_sb(conn);

                /* Update watched objects after detaching mark */
                if (sb)
                        fsnotify_update_sb_watchers(sb, conn);
                objp = fsnotify_recalc_mask_clear_iref(conn);
                type = conn->type;
        }
        WRITE_ONCE(mark->connector, NULL);
        spin_unlock(&conn->lock);

        fsnotify_drop_object(type, objp);

        if (free_conn) {
                spin_lock(&destroy_lock);
                conn->destroy_next = connector_destroy_list;
                connector_destroy_list = conn;
                spin_unlock(&destroy_lock);
                queue_work(system_dfl_wq, &connector_reaper_work);
        }
        /*
         * Note that we didn't update flags telling whether inode cares about
         * what's happening with children. We update these flags from
         * __fsnotify_parent() lazily when next event happens on one of our
         * children.
         */
        spin_lock(&destroy_lock);
        list_add(&mark->g_list, &destroy_list);
        spin_unlock(&destroy_lock);
        queue_delayed_work(system_dfl_wq, &reaper_work,
                           FSNOTIFY_REAPER_DELAY);
}
EXPORT_SYMBOL_GPL(fsnotify_put_mark);

/*
 * Get mark reference when we found the mark via lockless traversal of object
 * list. Mark can be already removed from the list by now and on its way to be
 * destroyed once SRCU period ends.
 *
 * Also pin the group so it doesn't disappear under us.
 */
static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark)
{
        if (refcount_inc_not_zero(&mark->refcnt)) {
                spin_lock(&mark->lock);
                if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) {
                        /* mark is attached, group is still alive then */
                        atomic_inc(&mark->group->user_waits);
                        spin_unlock(&mark->lock);
                        return true;
                }
                spin_unlock(&mark->lock);
                fsnotify_put_mark(mark);
        }
        return false;
}

/*
 * Puts marks and wakes up group destruction if necessary.
 *
 * Pairs with fsnotify_get_mark_safe()
 */
static void fsnotify_put_mark_wake(struct fsnotify_mark *mark)
{
        if (mark) {
                struct fsnotify_group *group = mark->group;

                fsnotify_put_mark(mark);
                /*
                 * We abuse notification_waitq on group shutdown for waiting for
                 * all marks pinned when waiting for userspace.
                 */
                if (atomic_dec_and_test(&group->user_waits) && group->shutdown)
                        wake_up(&group->notification_waitq);
        }
}

bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info)
        __releases(&fsnotify_mark_srcu)
{
        int type;

        fsnotify_foreach_iter_type(type) {
                struct fsnotify_mark *mark = iter_info->marks[type];

                /* This can fail if mark is being removed */
                while (mark && !fsnotify_get_mark_safe(mark)) {
                        if (mark->group == iter_info->current_group) {
                                __release(&fsnotify_mark_srcu);
                                goto fail;
                        }
                        /* This is a mark in an unrelated group, skip */
                        mark = fsnotify_next_mark(mark);
                        iter_info->marks[type] = mark;
                }
        }

        /*
         * Now that all marks are pinned by refcount in the inode / vfsmount / etc
         * lists, we can drop SRCU lock, and safely resume the list iteration
         * once userspace returns.
         */
        srcu_read_unlock(&fsnotify_mark_srcu, iter_info->srcu_idx);

        return true;

fail:
        for (type--; type >= 0; type--)
                fsnotify_put_mark_wake(iter_info->marks[type]);
        return false;
}

void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info)
        __acquires(&fsnotify_mark_srcu)
{
        int type;

        iter_info->srcu_idx = srcu_read_lock(&fsnotify_mark_srcu);
        fsnotify_foreach_iter_type(type)
                fsnotify_put_mark_wake(iter_info->marks[type]);
}

/*
 * Mark mark as detached, remove it from group list. Mark still stays in object
 * list until its last reference is dropped. Note that we rely on mark being
 * removed from group list before corresponding reference to it is dropped. In
 * particular we rely on mark->connector being valid while we hold
 * group->mark_mutex if we found the mark through g_list.
 *
 * Must be called with group->mark_mutex held. The caller must either hold
 * reference to the mark or be protected by fsnotify_mark_srcu.
 */
void fsnotify_detach_mark(struct fsnotify_mark *mark)
{
        fsnotify_group_assert_locked(mark->group);
        WARN_ON_ONCE(!srcu_read_lock_held(&fsnotify_mark_srcu) &&
                     refcount_read(&mark->refcnt) < 1 +
                        !!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED));

        spin_lock(&mark->lock);
        /* something else already called this function on this mark */
        if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
                spin_unlock(&mark->lock);
                return;
        }
        mark->flags &= ~FSNOTIFY_MARK_FLAG_ATTACHED;
        list_del_init(&mark->g_list);
        spin_unlock(&mark->lock);

        /* Drop mark reference acquired in fsnotify_add_mark_locked() */
        fsnotify_put_mark(mark);
}

/*
 * Free fsnotify mark. The mark is actually only marked as being freed.  The
 * freeing is actually happening only once last reference to the mark is
 * dropped from a workqueue which first waits for srcu period end.
 *
 * Caller must have a reference to the mark or be protected by
 * fsnotify_mark_srcu.
 */
void fsnotify_free_mark(struct fsnotify_mark *mark)
{
        struct fsnotify_group *group = mark->group;

        spin_lock(&mark->lock);
        /* something else already called this function on this mark */
        if (!(mark->flags & FSNOTIFY_MARK_FLAG_ALIVE)) {
                spin_unlock(&mark->lock);
                return;
        }
        mark->flags &= ~FSNOTIFY_MARK_FLAG_ALIVE;
        spin_unlock(&mark->lock);

        /*
         * Some groups like to know that marks are being freed.  This is a
         * callback to the group function to let it know that this mark
         * is being freed.
         */
        if (group->ops->freeing_mark)
                group->ops->freeing_mark(mark, group);
}

void fsnotify_destroy_mark(struct fsnotify_mark *mark,
                           struct fsnotify_group *group)
{
        fsnotify_group_lock(group);
        fsnotify_detach_mark(mark);
        fsnotify_group_unlock(group);
        fsnotify_free_mark(mark);
}
EXPORT_SYMBOL_GPL(fsnotify_destroy_mark);

/*
 * Sorting function for lists of fsnotify marks.
 *
 * Fanotify supports different notification classes (reflected as priority of
 * notification group). Events shall be passed to notification groups in
 * decreasing priority order. To achieve this marks in notification lists for
 * inodes and vfsmounts are sorted so that priorities of corresponding groups
 * are descending.
 *
 * Furthermore correct handling of the ignore mask requires processing inode
 * and vfsmount marks of each group together. Using the group address as
 * further sort criterion provides a unique sorting order and thus we can
 * merge inode and vfsmount lists of marks in linear time and find groups
 * present in both lists.
 *
 * A return value of 1 signifies that b has priority over a.
 * A return value of 0 signifies that the two marks have to be handled together.
 * A return value of -1 signifies that a has priority over b.
 */
int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b)
{
        if (a == b)
                return 0;
        if (!a)
                return 1;
        if (!b)
                return -1;
        if (a->priority < b->priority)
                return 1;
        if (a->priority > b->priority)
                return -1;
        if (a < b)
                return 1;
        return -1;
}

static int fsnotify_attach_info_to_sb(struct super_block *sb)
{
        struct fsnotify_sb_info *sbinfo;

        /* sb info is freed on fsnotify_sb_delete() */
        sbinfo = kzalloc_obj(*sbinfo);
        if (!sbinfo)
                return -ENOMEM;

        INIT_LIST_HEAD(&sbinfo->inode_conn_list);
        spin_lock_init(&sbinfo->list_lock);
        /*
         * cmpxchg() provides the barrier so that callers of fsnotify_sb_info()
         * will observe an initialized structure
         */
        if (cmpxchg(&sb->s_fsnotify_info, NULL, sbinfo)) {
                /* Someone else created sbinfo for us */
                kfree(sbinfo);
        }
        return 0;
}

struct fsnotify_inode_mark_connector {
        struct fsnotify_mark_connector common;
        struct list_head conns_list;
};

static struct inode *fsnotify_get_living_inode(struct fsnotify_sb_info *sbinfo)
{
        struct fsnotify_inode_mark_connector *iconn;
        struct inode *inode;

        spin_lock(&sbinfo->list_lock);
        /* Find the first non-evicting inode */
        list_for_each_entry(iconn, &sbinfo->inode_conn_list, conns_list) {
                /* All connectors on the list are still attached to an inode */
                inode = iconn->common.obj;
                /*
                 * For connectors without FSNOTIFY_CONN_FLAG_HAS_IREF
                 * (evictable marks) corresponding inode may well have 0
                 * refcount and can be undergoing eviction. OTOH list_lock
                 * protects us from the connector getting detached and inode
                 * freed. So we can poke around the inode safely.
                 */
                spin_lock(&inode->i_lock);
                if (likely(
                    !(inode_state_read(inode) & (I_FREEING | I_WILL_FREE)))) {
                        __iget(inode);
                        spin_unlock(&inode->i_lock);
                        spin_unlock(&sbinfo->list_lock);
                        return inode;
                }
                spin_unlock(&inode->i_lock);
        }
        spin_unlock(&sbinfo->list_lock);

        return NULL;
}

/**
 * fsnotify_unmount_inodes - an sb is unmounting. Handle any watched inodes.
 * @sbinfo: fsnotify info for superblock being unmounted.
 *
 * Walk all inode connectors for the superblock and free all associated marks.
 */
void fsnotify_unmount_inodes(struct fsnotify_sb_info *sbinfo)
{
        struct inode *inode;

        while ((inode = fsnotify_get_living_inode(sbinfo))) {
                fsnotify_inode(inode, FS_UNMOUNT);
                fsnotify_clear_marks_by_inode(inode);
                iput(inode);
                cond_resched();
        }
}

static void fsnotify_init_connector(struct fsnotify_mark_connector *conn,
                                    void *obj, unsigned int obj_type)
{
        spin_lock_init(&conn->lock);
        INIT_HLIST_HEAD(&conn->list);
        conn->flags = 0;
        conn->prio = 0;
        conn->type = obj_type;
        conn->obj = obj;
}

static struct fsnotify_mark_connector *
fsnotify_alloc_inode_connector(struct inode *inode)
{
        struct fsnotify_inode_mark_connector *iconn;
        struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(inode->i_sb);

        iconn = kmem_cache_alloc(fsnotify_inode_mark_connector_cachep,
                                 GFP_KERNEL);
        if (!iconn)
                return NULL;

        fsnotify_init_connector(&iconn->common, inode, FSNOTIFY_OBJ_TYPE_INODE);
        spin_lock(&sbinfo->list_lock);
        list_add(&iconn->conns_list, &sbinfo->inode_conn_list);
        spin_unlock(&sbinfo->list_lock);

        return &iconn->common;
}

static void fsnotify_untrack_connector(struct fsnotify_mark_connector *conn)
{
        struct fsnotify_inode_mark_connector *iconn;
        struct fsnotify_sb_info *sbinfo;

        if (conn->type != FSNOTIFY_OBJ_TYPE_INODE)
                return;

        iconn = container_of(conn, struct fsnotify_inode_mark_connector, common);
        sbinfo = fsnotify_sb_info(fsnotify_conn_inode(conn)->i_sb);
        spin_lock(&sbinfo->list_lock);
        list_del(&iconn->conns_list);
        spin_unlock(&sbinfo->list_lock);
}

static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
                                               void *obj, unsigned int obj_type)
{
        struct fsnotify_mark_connector *conn;

        if (obj_type == FSNOTIFY_OBJ_TYPE_INODE) {
                struct inode *inode = obj;

                conn = fsnotify_alloc_inode_connector(inode);
        } else {
                conn = kmem_cache_alloc(fsnotify_mark_connector_cachep,
                                        GFP_KERNEL);
                if (conn)
                        fsnotify_init_connector(conn, obj, obj_type);
        }
        if (!conn)
                return -ENOMEM;

        /*
         * cmpxchg() provides the barrier so that readers of *connp can see
         * only initialized structure
         */
        if (cmpxchg(connp, NULL, conn)) {
                /* Someone else created list structure for us */
                fsnotify_untrack_connector(conn);
                kfree(conn);
        }
        return 0;
}

/*
 * Get mark connector, make sure it is alive and return with its lock held.
 * This is for users that get connector pointer from inode or mount. Users that
 * hold reference to a mark on the list may directly lock connector->lock as
 * they are sure list cannot go away under them.
 */
static struct fsnotify_mark_connector *fsnotify_grab_connector(
                                                fsnotify_connp_t *connp)
{
        struct fsnotify_mark_connector *conn;
        int idx;

        idx = srcu_read_lock(&fsnotify_mark_srcu);
        conn = srcu_dereference(*connp, &fsnotify_mark_srcu);
        if (!conn)
                goto out;
        spin_lock(&conn->lock);
        if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) {
                spin_unlock(&conn->lock);
                srcu_read_unlock(&fsnotify_mark_srcu, idx);
                return NULL;
        }
out:
        srcu_read_unlock(&fsnotify_mark_srcu, idx);
        return conn;
}

/*
 * Add mark into proper place in given list of marks. These marks may be used
 * for the fsnotify backend to determine which event types should be delivered
 * to which group and for which inodes. These marks are ordered according to
 * priority, highest number first, and then by the group's location in memory.
 */
static int fsnotify_add_mark_list(struct fsnotify_mark *mark, void *obj,
                                  unsigned int obj_type, int add_flags)
{
        struct super_block *sb = fsnotify_object_sb(obj, obj_type);
        struct fsnotify_mark *lmark, *last = NULL;
        struct fsnotify_mark_connector *conn;
        fsnotify_connp_t *connp;
        int cmp;
        int err = 0;

        if (WARN_ON(!fsnotify_valid_obj_type(obj_type)))
                return -EINVAL;

        /*
         * Attach the sb info before attaching a connector to any object on sb.
         * The sb info will remain attached as long as sb lives.
         */
        if (sb && !fsnotify_sb_info(sb)) {
                err = fsnotify_attach_info_to_sb(sb);
                if (err)
                        return err;
        }

        connp = fsnotify_object_connp(obj, obj_type);
restart:
        spin_lock(&mark->lock);
        conn = fsnotify_grab_connector(connp);
        if (!conn) {
                spin_unlock(&mark->lock);
                err = fsnotify_attach_connector_to_object(connp, obj, obj_type);
                if (err)
                        return err;
                goto restart;
        }

        /* is mark the first mark? */
        if (hlist_empty(&conn->list)) {
                hlist_add_head_rcu(&mark->obj_list, &conn->list);
                goto added;
        }

        /* should mark be in the middle of the current list? */
        hlist_for_each_entry(lmark, &conn->list, obj_list) {
                last = lmark;

                if ((lmark->group == mark->group) &&
                    (lmark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) &&
                    !(mark->group->flags & FSNOTIFY_GROUP_DUPS)) {
                        err = -EEXIST;
                        goto out_err;
                }

                cmp = fsnotify_compare_groups(lmark->group, mark->group);
                if (cmp >= 0) {
                        hlist_add_before_rcu(&mark->obj_list, &lmark->obj_list);
                        goto added;
                }
        }

        BUG_ON(last == NULL);
        /* mark should be the last entry.  last is the current last entry */
        hlist_add_behind_rcu(&mark->obj_list, &last->obj_list);
added:
        if (sb)
                fsnotify_update_sb_watchers(sb, conn);
        /*
         * Since connector is attached to object using cmpxchg() we are
         * guaranteed that connector initialization is fully visible by anyone
         * seeing mark->connector set.
         */
        WRITE_ONCE(mark->connector, conn);
out_err:
        spin_unlock(&conn->lock);
        spin_unlock(&mark->lock);
        return err;
}

/*
 * Attach an initialized mark to a given group and fs object.
 * These marks may be used for the fsnotify backend to determine which
 * event types should be delivered to which group.
 */
int fsnotify_add_mark_locked(struct fsnotify_mark *mark,
                             void *obj, unsigned int obj_type,
                             int add_flags)
{
        struct fsnotify_group *group = mark->group;
        int ret = 0;

        fsnotify_group_assert_locked(group);

        /*
         * LOCKING ORDER!!!!
         * group->mark_mutex
         * mark->lock
         * mark->connector->lock
         */
        spin_lock(&mark->lock);
        mark->flags |= FSNOTIFY_MARK_FLAG_ALIVE | FSNOTIFY_MARK_FLAG_ATTACHED;

        list_add(&mark->g_list, &group->marks_list);
        fsnotify_get_mark(mark); /* for g_list */
        spin_unlock(&mark->lock);

        ret = fsnotify_add_mark_list(mark, obj, obj_type, add_flags);
        if (ret)
                goto err;

        fsnotify_recalc_mask(mark->connector);

        return ret;
err:
        spin_lock(&mark->lock);
        mark->flags &= ~(FSNOTIFY_MARK_FLAG_ALIVE |
                         FSNOTIFY_MARK_FLAG_ATTACHED);
        list_del_init(&mark->g_list);
        spin_unlock(&mark->lock);

        fsnotify_put_mark(mark);
        return ret;
}

int fsnotify_add_mark(struct fsnotify_mark *mark, void *obj,
                      unsigned int obj_type, int add_flags)
{
        int ret;
        struct fsnotify_group *group = mark->group;

        fsnotify_group_lock(group);
        ret = fsnotify_add_mark_locked(mark, obj, obj_type, add_flags);
        fsnotify_group_unlock(group);
        return ret;
}
EXPORT_SYMBOL_GPL(fsnotify_add_mark);

/*
 * Given a list of marks, find the mark associated with given group. If found
 * take a reference to that mark and return it, else return NULL.
 */
struct fsnotify_mark *fsnotify_find_mark(void *obj, unsigned int obj_type,
                                         struct fsnotify_group *group)
{
        fsnotify_connp_t *connp = fsnotify_object_connp(obj, obj_type);
        struct fsnotify_mark_connector *conn;
        struct fsnotify_mark *mark;

        if (!connp)
                return NULL;

        conn = fsnotify_grab_connector(connp);
        if (!conn)
                return NULL;

        hlist_for_each_entry(mark, &conn->list, obj_list) {
                if (mark->group == group &&
                    (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) {
                        fsnotify_get_mark(mark);
                        spin_unlock(&conn->lock);
                        return mark;
                }
        }
        spin_unlock(&conn->lock);
        return NULL;
}
EXPORT_SYMBOL_GPL(fsnotify_find_mark);

/* Clear any marks in a group with given type mask */
void fsnotify_clear_marks_by_group(struct fsnotify_group *group,
                                   unsigned int obj_type)
{
        struct fsnotify_mark *lmark, *mark;
        LIST_HEAD(to_free);
        struct list_head *head = &to_free;

        /* Skip selection step if we want to clear all marks. */
        if (obj_type == FSNOTIFY_OBJ_TYPE_ANY) {
                head = &group->marks_list;
                goto clear;
        }
        /*
         * We have to be really careful here. Anytime we drop mark_mutex, e.g.
         * fsnotify_clear_marks_by_inode() can come and free marks. Even in our
         * to_free list so we have to use mark_mutex even when accessing that
         * list. And freeing mark requires us to drop mark_mutex. So we can
         * reliably free only the first mark in the list. That's why we first
         * move marks to free to to_free list in one go and then free marks in
         * to_free list one by one.
         */
        fsnotify_group_lock(group);
        list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) {
                if (mark->connector->type == obj_type)
                        list_move(&mark->g_list, &to_free);
        }
        fsnotify_group_unlock(group);

clear:
        while (1) {
                fsnotify_group_lock(group);
                if (list_empty(head)) {
                        fsnotify_group_unlock(group);
                        break;
                }
                mark = list_first_entry(head, struct fsnotify_mark, g_list);
                fsnotify_get_mark(mark);
                fsnotify_detach_mark(mark);
                fsnotify_group_unlock(group);
                fsnotify_free_mark(mark);
                fsnotify_put_mark(mark);
        }
}

/* Destroy all marks attached to an object via connector */
void fsnotify_destroy_marks(fsnotify_connp_t *connp)
{
        struct fsnotify_mark_connector *conn;
        struct fsnotify_mark *mark, *old_mark = NULL;
        void *objp;
        unsigned int type;

        conn = fsnotify_grab_connector(connp);
        if (!conn)
                return;
        /*
         * We have to be careful since we can race with e.g.
         * fsnotify_clear_marks_by_group() and once we drop the conn->lock, the
         * list can get modified. However we are holding mark reference and
         * thus our mark cannot be removed from obj_list so we can continue
         * iteration after regaining conn->lock.
         */
        hlist_for_each_entry(mark, &conn->list, obj_list) {
                fsnotify_get_mark(mark);
                spin_unlock(&conn->lock);
                if (old_mark)
                        fsnotify_put_mark(old_mark);
                old_mark = mark;
                fsnotify_destroy_mark(mark, mark->group);
                spin_lock(&conn->lock);
        }
        /*
         * Detach list from object now so that we don't pin inode until all
         * mark references get dropped. It would lead to strange results such
         * as delaying inode deletion or blocking unmount.
         */
        objp = fsnotify_detach_connector_from_object(conn, &type);
        spin_unlock(&conn->lock);
        if (old_mark)
                fsnotify_put_mark(old_mark);
        fsnotify_drop_object(type, objp);
}

/*
 * Nothing fancy, just initialize lists and locks and counters.
 */
void fsnotify_init_mark(struct fsnotify_mark *mark,
                        struct fsnotify_group *group)
{
        memset(mark, 0, sizeof(*mark));
        spin_lock_init(&mark->lock);
        refcount_set(&mark->refcnt, 1);
        fsnotify_get_group(group);
        mark->group = group;
        WRITE_ONCE(mark->connector, NULL);
}
EXPORT_SYMBOL_GPL(fsnotify_init_mark);

/*
 * Destroy all marks in destroy_list, waits for SRCU period to finish before
 * actually freeing marks.
 */
static void fsnotify_mark_destroy_workfn(struct work_struct *work)
{
        struct fsnotify_mark *mark, *next;
        struct list_head private_destroy_list;

        spin_lock(&destroy_lock);
        /* exchange the list head */
        list_replace_init(&destroy_list, &private_destroy_list);
        spin_unlock(&destroy_lock);

        synchronize_srcu(&fsnotify_mark_srcu);

        list_for_each_entry_safe(mark, next, &private_destroy_list, g_list) {
                list_del_init(&mark->g_list);
                fsnotify_final_mark_destroy(mark);
        }
}

/* Wait for all marks queued for destruction to be actually destroyed */
void fsnotify_wait_marks_destroyed(void)
{
        flush_delayed_work(&reaper_work);
}
EXPORT_SYMBOL_GPL(fsnotify_wait_marks_destroyed);

__init void fsnotify_init_connector_caches(void)
{
        fsnotify_mark_connector_cachep = KMEM_CACHE(fsnotify_mark_connector,
                                                    SLAB_PANIC);
        fsnotify_inode_mark_connector_cachep = KMEM_CACHE(
                                        fsnotify_inode_mark_connector,
                                        SLAB_PANIC);
}

























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
/* SPDX-License-Identifier: GPL-2.0 */
/*
 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 *  Definitions for LLC (link layer control) message handling
 *
 *  Copyright IBM Corp. 2016
 *
 *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
 *              Ursula Braun <ubraun@linux.vnet.ibm.com>
 */

#ifndef SMC_LLC_H
#define SMC_LLC_H

#include "smc_wr.h"

#define SMC_LLC_FLAG_RESP                0x80

#define SMC_LLC_WAIT_FIRST_TIME                (5 * HZ)
#define SMC_LLC_WAIT_TIME                (2 * HZ)
#define SMC_LLC_TESTLINK_DEFAULT_TIME        (30 * HZ)

enum smc_llc_reqresp {
        SMC_LLC_REQ,
        SMC_LLC_RESP
};

enum smc_llc_msg_type {
        SMC_LLC_CONFIRM_LINK                = 0x01,
        SMC_LLC_ADD_LINK                = 0x02,
        SMC_LLC_ADD_LINK_CONT                = 0x03,
        SMC_LLC_DELETE_LINK                = 0x04,
        SMC_LLC_REQ_ADD_LINK                = 0x05,
        SMC_LLC_CONFIRM_RKEY                = 0x06,
        SMC_LLC_TEST_LINK                = 0x07,
        SMC_LLC_CONFIRM_RKEY_CONT        = 0x08,
        SMC_LLC_DELETE_RKEY                = 0x09,
        /* V2 types */
        SMC_LLC_CONFIRM_LINK_V2                = 0x21,
        SMC_LLC_ADD_LINK_V2                = 0x22,
        SMC_LLC_DELETE_LINK_V2                = 0x24,
        SMC_LLC_REQ_ADD_LINK_V2                = 0x25,
        SMC_LLC_CONFIRM_RKEY_V2                = 0x26,
        SMC_LLC_TEST_LINK_V2                = 0x27,
        SMC_LLC_DELETE_RKEY_V2                = 0x29,
};

#define smc_link_downing(state) \
        (cmpxchg(state, SMC_LNK_ACTIVE, SMC_LNK_INACTIVE) == SMC_LNK_ACTIVE)

/* LLC DELETE LINK Request Reason Codes */
#define SMC_LLC_DEL_LOST_PATH                0x00010000
#define SMC_LLC_DEL_OP_INIT_TERM        0x00020000
#define SMC_LLC_DEL_PROG_INIT_TERM        0x00030000
#define SMC_LLC_DEL_PROT_VIOL                0x00040000
#define SMC_LLC_DEL_NO_ASYM_NEEDED        0x00050000
/* LLC DELETE LINK Response Reason Codes */
#define SMC_LLC_DEL_NOLNK        0x00100000  /* Unknown Link ID (no link) */
#define SMC_LLC_DEL_NOLGR        0x00200000  /* Unknown Link Group */

/* returns a usable link of the link group, or NULL */
static inline struct smc_link *smc_llc_usable_link(struct smc_link_group *lgr)
{
        int i;

        for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
                if (smc_link_usable(&lgr->lnk[i]))
                        return &lgr->lnk[i];
        return NULL;
}

/* set the termination reason code for the link group */
static inline void smc_llc_set_termination_rsn(struct smc_link_group *lgr,
                                               u32 rsn)
{
        if (!lgr->llc_termination_rsn)
                lgr->llc_termination_rsn = rsn;
}

/* transmit */
int smc_llc_send_confirm_link(struct smc_link *lnk,
                              enum smc_llc_reqresp reqresp);
int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
                          struct smc_link *link_new,
                          enum smc_llc_reqresp reqresp);
int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
                             enum smc_llc_reqresp reqresp, bool orderly,
                             u32 reason);
void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id);
void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc);
void smc_llc_lgr_clear(struct smc_link_group *lgr);
int smc_llc_link_init(struct smc_link *link);
void smc_llc_link_active(struct smc_link *link);
void smc_llc_link_clear(struct smc_link *link, bool log);
int smc_llc_do_confirm_rkey(struct smc_link *send_link,
                            struct smc_buf_desc *rmb_desc);
int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
                           struct smc_buf_desc *rmb_desc);
int smc_llc_flow_initiate(struct smc_link_group *lgr,
                          enum smc_llc_flowtype type);
void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow);
int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
                           enum smc_llc_reqresp type);
void smc_llc_link_set_uid(struct smc_link *link);
void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry);
struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
                                    struct smc_link *lnk,
                                    int time_out, u8 exp_msg);
struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow);
void smc_llc_flow_qentry_del(struct smc_llc_flow *flow);
void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord,
                                  u32 rsn);
int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry);
int smc_llc_srv_add_link(struct smc_link *link,
                         struct smc_llc_qentry *req_qentry);
void smc_llc_add_link_local(struct smc_link *link);
int smc_llc_init(void) __init;

#endif /* SMC_LLC_H */
























































































































































































































































































































































































































































   30 










   30 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
/*
 * proc.c - procfs support for Protocol family CAN core module
 *
 * Copyright (c) 2002-2007 Volkswagen Group Electronic Research
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of Volkswagen nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * Alternatively, provided that this notice is retained in full, this
 * software may be distributed under the terms of the GNU General
 * Public License ("GPL") version 2, in which case the provisions of the
 * GPL apply INSTEAD OF those given above.
 *
 * The provided data structures and external interfaces from this code
 * are not restricted to be used by modules with a GPL compatible license.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 *
 */

#include <linux/module.h>
#include <linux/proc_fs.h>
#include <linux/list.h>
#include <linux/rcupdate.h>
#include <linux/if_arp.h>
#include <linux/can/can-ml.h>
#include <linux/can/core.h>

#include "af_can.h"

/*
 * proc filenames for the PF_CAN core
 */

#define CAN_PROC_STATS       "stats"
#define CAN_PROC_RESET_STATS "reset_stats"
#define CAN_PROC_RCVLIST_ALL "rcvlist_all"
#define CAN_PROC_RCVLIST_FIL "rcvlist_fil"
#define CAN_PROC_RCVLIST_INV "rcvlist_inv"
#define CAN_PROC_RCVLIST_SFF "rcvlist_sff"
#define CAN_PROC_RCVLIST_EFF "rcvlist_eff"
#define CAN_PROC_RCVLIST_ERR "rcvlist_err"

static int user_reset;

static const char rx_list_name[][8] = {
        [RX_ERR] = "rx_err",
        [RX_ALL] = "rx_all",
        [RX_FIL] = "rx_fil",
        [RX_INV] = "rx_inv",
};

/*
 * af_can statistics stuff
 */

static void can_init_stats(struct net *net)
{
        struct can_pkg_stats *pkg_stats = net->can.pkg_stats;
        struct can_rcv_lists_stats *rcv_lists_stats = net->can.rcv_lists_stats;
        /*
         * This memset function is called from a timer context (when
         * can_stattimer is active which is the default) OR in a process
         * context (reading the proc_fs when can_stattimer is disabled).
         */
        memset(pkg_stats, 0, sizeof(struct can_pkg_stats));
        pkg_stats->jiffies_init = jiffies;

        rcv_lists_stats->stats_reset++;

        if (user_reset) {
                user_reset = 0;
                rcv_lists_stats->user_reset++;
        }
}

static unsigned long calc_rate(unsigned long oldjif, unsigned long newjif,
                               unsigned long count)
{
        if (oldjif == newjif)
                return 0;

        /* see can_stat_update() - this should NEVER happen! */
        if (count > (ULONG_MAX / HZ)) {
                printk(KERN_ERR "can: calc_rate: count exceeded! %ld\n",
                       count);
                return 99999999;
        }

        return (count * HZ) / (newjif - oldjif);
}

void can_stat_update(struct timer_list *t)
{
        struct net *net = timer_container_of(net, t, can.stattimer);
        struct can_pkg_stats *pkg_stats = net->can.pkg_stats;
        unsigned long j = jiffies; /* snapshot */

        long rx_frames = atomic_long_read(&pkg_stats->rx_frames);
        long tx_frames = atomic_long_read(&pkg_stats->tx_frames);
        long matches = atomic_long_read(&pkg_stats->matches);
        long rx_frames_delta = atomic_long_read(&pkg_stats->rx_frames_delta);
        long tx_frames_delta = atomic_long_read(&pkg_stats->tx_frames_delta);
        long matches_delta = atomic_long_read(&pkg_stats->matches_delta);

        /* restart counting in timer context on user request */
        if (user_reset)
                can_init_stats(net);

        /* restart counting on jiffies overflow */
        if (j < pkg_stats->jiffies_init)
                can_init_stats(net);

        /* prevent overflow in calc_rate() */
        if (rx_frames > (LONG_MAX / HZ))
                can_init_stats(net);

        /* prevent overflow in calc_rate() */
        if (tx_frames > (LONG_MAX / HZ))
                can_init_stats(net);

        /* matches overflow - very improbable */
        if (matches > (LONG_MAX / 100))
                can_init_stats(net);

        /* calc total values */
        if (rx_frames)
                pkg_stats->total_rx_match_ratio = (matches * 100) / rx_frames;

        pkg_stats->total_tx_rate = calc_rate(pkg_stats->jiffies_init, j,
                                            tx_frames);
        pkg_stats->total_rx_rate = calc_rate(pkg_stats->jiffies_init, j,
                                            rx_frames);

        /* calc current values */
        if (rx_frames_delta)
                pkg_stats->current_rx_match_ratio =
                        (matches_delta * 100) /        rx_frames_delta;

        pkg_stats->current_tx_rate = calc_rate(0, HZ, tx_frames_delta);
        pkg_stats->current_rx_rate = calc_rate(0, HZ, rx_frames_delta);

        /* check / update maximum values */
        if (pkg_stats->max_tx_rate < pkg_stats->current_tx_rate)
                pkg_stats->max_tx_rate = pkg_stats->current_tx_rate;

        if (pkg_stats->max_rx_rate < pkg_stats->current_rx_rate)
                pkg_stats->max_rx_rate = pkg_stats->current_rx_rate;

        if (pkg_stats->max_rx_match_ratio < pkg_stats->current_rx_match_ratio)
                pkg_stats->max_rx_match_ratio = pkg_stats->current_rx_match_ratio;

        /* clear values for 'current rate' calculation */
        atomic_long_set(&pkg_stats->tx_frames_delta, 0);
        atomic_long_set(&pkg_stats->rx_frames_delta, 0);
        atomic_long_set(&pkg_stats->matches_delta, 0);

        /* restart timer (one second) */
        mod_timer(&net->can.stattimer, round_jiffies(jiffies + HZ));
}

/*
 * proc read functions
 */

static void can_print_rcvlist(struct seq_file *m, struct hlist_head *rx_list,
                              struct net_device *dev)
{
        struct receiver *r;

        hlist_for_each_entry_rcu(r, rx_list, list) {
                char *fmt = (r->can_id & CAN_EFF_FLAG)?
                        "   %-5s  %08x  %08x  %pK  %pK  %8ld  %s\n" :
                        "   %-5s     %03x    %08x  %pK  %pK  %8ld  %s\n";

                seq_printf(m, fmt, DNAME(dev), r->can_id, r->mask,
                           r->func, r->data, atomic_long_read(&r->matches),
                           r->ident);
        }
}

static void can_print_recv_banner(struct seq_file *m)
{
        /*
         *                  can1.  00000000  00000000  00000000
         *                 .......          0  tp20
         */
        if (IS_ENABLED(CONFIG_64BIT))
                seq_puts(m, "  device   can_id   can_mask      function          userdata       matches  ident\n");
        else
                seq_puts(m, "  device   can_id   can_mask  function  userdata   matches  ident\n");
}

static int can_stats_proc_show(struct seq_file *m, void *v)
{
        struct net *net = m->private;
        struct can_pkg_stats *pkg_stats = net->can.pkg_stats;
        struct can_rcv_lists_stats *rcv_lists_stats = net->can.rcv_lists_stats;

        seq_putc(m, '\n');
        seq_printf(m, " %8ld transmitted frames (TXF)\n",
                   atomic_long_read(&pkg_stats->tx_frames));
        seq_printf(m, " %8ld received frames (RXF)\n",
                   atomic_long_read(&pkg_stats->rx_frames));
        seq_printf(m, " %8ld matched frames (RXMF)\n",
                   atomic_long_read(&pkg_stats->matches));

        seq_putc(m, '\n');

        if (net->can.stattimer.function == can_stat_update) {
                seq_printf(m, " %8ld %% total match ratio (RXMR)\n",
                                pkg_stats->total_rx_match_ratio);

                seq_printf(m, " %8ld frames/s total tx rate (TXR)\n",
                                pkg_stats->total_tx_rate);
                seq_printf(m, " %8ld frames/s total rx rate (RXR)\n",
                                pkg_stats->total_rx_rate);

                seq_putc(m, '\n');

                seq_printf(m, " %8ld %% current match ratio (CRXMR)\n",
                                pkg_stats->current_rx_match_ratio);

                seq_printf(m, " %8ld frames/s current tx rate (CTXR)\n",
                                pkg_stats->current_tx_rate);
                seq_printf(m, " %8ld frames/s current rx rate (CRXR)\n",
                                pkg_stats->current_rx_rate);

                seq_putc(m, '\n');

                seq_printf(m, " %8ld %% max match ratio (MRXMR)\n",
                                pkg_stats->max_rx_match_ratio);

                seq_printf(m, " %8ld frames/s max tx rate (MTXR)\n",
                                pkg_stats->max_tx_rate);
                seq_printf(m, " %8ld frames/s max rx rate (MRXR)\n",
                                pkg_stats->max_rx_rate);

                seq_putc(m, '\n');
        }

        seq_printf(m, " %8ld current receive list entries (CRCV)\n",
                        rcv_lists_stats->rcv_entries);
        seq_printf(m, " %8ld maximum receive list entries (MRCV)\n",
                        rcv_lists_stats->rcv_entries_max);

        if (rcv_lists_stats->stats_reset)
                seq_printf(m, "\n %8ld statistic resets (STR)\n",
                                rcv_lists_stats->stats_reset);

        if (rcv_lists_stats->user_reset)
                seq_printf(m, " %8ld user statistic resets (USTR)\n",
                                rcv_lists_stats->user_reset);

        seq_putc(m, '\n');
        return 0;
}

static int can_reset_stats_proc_show(struct seq_file *m, void *v)
{
        struct net *net = m->private;
        struct can_rcv_lists_stats *rcv_lists_stats = net->can.rcv_lists_stats;
        struct can_pkg_stats *pkg_stats = net->can.pkg_stats;

        user_reset = 1;

        if (net->can.stattimer.function == can_stat_update) {
                seq_printf(m, "Scheduled statistic reset #%ld.\n",
                                rcv_lists_stats->stats_reset + 1);
        } else {
                if (pkg_stats->jiffies_init != jiffies)
                        can_init_stats(net);

                seq_printf(m, "Performed statistic reset #%ld.\n",
                                rcv_lists_stats->stats_reset);
        }
        return 0;
}

static inline void can_rcvlist_proc_show_one(struct seq_file *m, int idx,
                                             struct net_device *dev,
                                             struct can_dev_rcv_lists *dev_rcv_lists)
{
        if (!hlist_empty(&dev_rcv_lists->rx[idx])) {
                can_print_recv_banner(m);
                can_print_rcvlist(m, &dev_rcv_lists->rx[idx], dev);
        } else
                seq_printf(m, "  (%s: no entry)\n", DNAME(dev));

}

static int can_rcvlist_proc_show(struct seq_file *m, void *v)
{
        /* double cast to prevent GCC warning */
        int idx = (int)(long)pde_data(m->file->f_inode);
        struct net_device *dev;
        struct can_dev_rcv_lists *dev_rcv_lists;
        struct net *net = m->private;

        seq_printf(m, "\nreceive list '%s':\n", rx_list_name[idx]);

        rcu_read_lock();

        /* receive list for 'all' CAN devices (dev == NULL) */
        dev_rcv_lists = net->can.rx_alldev_list;
        can_rcvlist_proc_show_one(m, idx, NULL, dev_rcv_lists);

        /* receive list for registered CAN devices */
        for_each_netdev_rcu(net, dev) {
                struct can_ml_priv *can_ml = can_get_ml_priv(dev);

                if (can_ml)
                        can_rcvlist_proc_show_one(m, idx, dev,
                                                  &can_ml->dev_rcv_lists);
        }

        rcu_read_unlock();

        seq_putc(m, '\n');
        return 0;
}

static inline void can_rcvlist_proc_show_array(struct seq_file *m,
                                               struct net_device *dev,
                                               struct hlist_head *rcv_array,
                                               unsigned int rcv_array_sz)
{
        unsigned int i;
        int all_empty = 1;

        /* check whether at least one list is non-empty */
        for (i = 0; i < rcv_array_sz; i++)
                if (!hlist_empty(&rcv_array[i])) {
                        all_empty = 0;
                        break;
                }

        if (!all_empty) {
                can_print_recv_banner(m);
                for (i = 0; i < rcv_array_sz; i++) {
                        if (!hlist_empty(&rcv_array[i]))
                                can_print_rcvlist(m, &rcv_array[i], dev);
                }
        } else
                seq_printf(m, "  (%s: no entry)\n", DNAME(dev));
}

static int can_rcvlist_sff_proc_show(struct seq_file *m, void *v)
{
        struct net_device *dev;
        struct can_dev_rcv_lists *dev_rcv_lists;
        struct net *net = m->private;

        /* RX_SFF */
        seq_puts(m, "\nreceive list 'rx_sff':\n");

        rcu_read_lock();

        /* sff receive list for 'all' CAN devices (dev == NULL) */
        dev_rcv_lists = net->can.rx_alldev_list;
        can_rcvlist_proc_show_array(m, NULL, dev_rcv_lists->rx_sff,
                                    ARRAY_SIZE(dev_rcv_lists->rx_sff));

        /* sff receive list for registered CAN devices */
        for_each_netdev_rcu(net, dev) {
                struct can_ml_priv *can_ml = can_get_ml_priv(dev);

                if (can_ml) {
                        dev_rcv_lists = &can_ml->dev_rcv_lists;
                        can_rcvlist_proc_show_array(m, dev, dev_rcv_lists->rx_sff,
                                                    ARRAY_SIZE(dev_rcv_lists->rx_sff));
                }
        }

        rcu_read_unlock();

        seq_putc(m, '\n');
        return 0;
}

static int can_rcvlist_eff_proc_show(struct seq_file *m, void *v)
{
        struct net_device *dev;
        struct can_dev_rcv_lists *dev_rcv_lists;
        struct net *net = m->private;

        /* RX_EFF */
        seq_puts(m, "\nreceive list 'rx_eff':\n");

        rcu_read_lock();

        /* eff receive list for 'all' CAN devices (dev == NULL) */
        dev_rcv_lists = net->can.rx_alldev_list;
        can_rcvlist_proc_show_array(m, NULL, dev_rcv_lists->rx_eff,
                                    ARRAY_SIZE(dev_rcv_lists->rx_eff));

        /* eff receive list for registered CAN devices */
        for_each_netdev_rcu(net, dev) {
                struct can_ml_priv *can_ml = can_get_ml_priv(dev);

                if (can_ml) {
                        dev_rcv_lists = &can_ml->dev_rcv_lists;
                        can_rcvlist_proc_show_array(m, dev, dev_rcv_lists->rx_eff,
                                                    ARRAY_SIZE(dev_rcv_lists->rx_eff));
                }
        }

        rcu_read_unlock();

        seq_putc(m, '\n');
        return 0;
}

/*
 * can_init_proc - create main CAN proc directory and procfs entries
 */
void can_init_proc(struct net *net)
{
        /* create /proc/net/can directory */
        net->can.proc_dir = proc_net_mkdir(net, "can", net->proc_net);

        if (!net->can.proc_dir) {
                printk(KERN_INFO "can: failed to create /proc/net/can . "
                           "CONFIG_PROC_FS missing?\n");
                return;
        }

        /* own procfs entries from the AF_CAN core */
        net->can.pde_stats = proc_create_net_single(CAN_PROC_STATS, 0644,
                        net->can.proc_dir, can_stats_proc_show, NULL);
        net->can.pde_reset_stats = proc_create_net_single(CAN_PROC_RESET_STATS,
                        0644, net->can.proc_dir, can_reset_stats_proc_show,
                        NULL);
        net->can.pde_rcvlist_err = proc_create_net_single(CAN_PROC_RCVLIST_ERR,
                        0644, net->can.proc_dir, can_rcvlist_proc_show,
                        (void *)RX_ERR);
        net->can.pde_rcvlist_all = proc_create_net_single(CAN_PROC_RCVLIST_ALL,
                        0644, net->can.proc_dir, can_rcvlist_proc_show,
                        (void *)RX_ALL);
        net->can.pde_rcvlist_fil = proc_create_net_single(CAN_PROC_RCVLIST_FIL,
                        0644, net->can.proc_dir, can_rcvlist_proc_show,
                        (void *)RX_FIL);
        net->can.pde_rcvlist_inv = proc_create_net_single(CAN_PROC_RCVLIST_INV,
                        0644, net->can.proc_dir, can_rcvlist_proc_show,
                        (void *)RX_INV);
        net->can.pde_rcvlist_eff = proc_create_net_single(CAN_PROC_RCVLIST_EFF,
                        0644, net->can.proc_dir, can_rcvlist_eff_proc_show, NULL);
        net->can.pde_rcvlist_sff = proc_create_net_single(CAN_PROC_RCVLIST_SFF,
                        0644, net->can.proc_dir, can_rcvlist_sff_proc_show, NULL);
}

/*
 * can_remove_proc - remove procfs entries and main CAN proc directory
 */
void can_remove_proc(struct net *net)
{
        if (!net->can.proc_dir)
                return;

        if (net->can.pde_stats)
                remove_proc_entry(CAN_PROC_STATS, net->can.proc_dir);

        if (net->can.pde_reset_stats)
                remove_proc_entry(CAN_PROC_RESET_STATS, net->can.proc_dir);

        if (net->can.pde_rcvlist_err)
                remove_proc_entry(CAN_PROC_RCVLIST_ERR, net->can.proc_dir);

        if (net->can.pde_rcvlist_all)
                remove_proc_entry(CAN_PROC_RCVLIST_ALL, net->can.proc_dir);

        if (net->can.pde_rcvlist_fil)
                remove_proc_entry(CAN_PROC_RCVLIST_FIL, net->can.proc_dir);

        if (net->can.pde_rcvlist_inv)
                remove_proc_entry(CAN_PROC_RCVLIST_INV, net->can.proc_dir);

        if (net->can.pde_rcvlist_eff)
                remove_proc_entry(CAN_PROC_RCVLIST_EFF, net->can.proc_dir);

        if (net->can.pde_rcvlist_sff)
                remove_proc_entry(CAN_PROC_RCVLIST_SFF, net->can.proc_dir);

        remove_proc_entry("can", net->proc_net);
}







































































































  571 





  570 

  571 
  570 
  572 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        common UDP/RAW code
 *        Linux INET implementation
 *
 * Authors:
 *         Hideaki YOSHIFUJI <yoshfuji@linux-ipv6.org>
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/in.h>
#include <net/ip.h>
#include <net/sock.h>
#include <net/route.h>
#include <net/tcp_states.h>
#include <net/sock_reuseport.h>

int __ip4_datagram_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len)
{
        struct inet_sock *inet = inet_sk(sk);
        struct sockaddr_in *usin = (struct sockaddr_in *) uaddr;
        struct flowi4 *fl4;
        struct rtable *rt;
        __be32 saddr;
        int oif;
        int err;


        if (addr_len < sizeof(*usin))
                return -EINVAL;

        if (usin->sin_family != AF_INET)
                return -EAFNOSUPPORT;

        sk_dst_reset(sk);

        oif = sk->sk_bound_dev_if;
        saddr = inet->inet_saddr;
        if (ipv4_is_multicast(usin->sin_addr.s_addr)) {
                if (!oif || netif_index_is_l3_master(sock_net(sk), oif))
                        oif = READ_ONCE(inet->mc_index);
                if (!saddr)
                        saddr = READ_ONCE(inet->mc_addr);
        } else if (!oif) {
                oif = READ_ONCE(inet->uc_index);
        }
        fl4 = &inet->cork.fl.u.ip4;
        rt = ip_route_connect(fl4, usin->sin_addr.s_addr, saddr, oif,
                              sk->sk_protocol, inet->inet_sport,
                              usin->sin_port, sk);
        if (IS_ERR(rt)) {
                err = PTR_ERR(rt);
                if (err == -ENETUNREACH)
                        IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
                goto out;
        }

        if ((rt->rt_flags & RTCF_BROADCAST) && !sock_flag(sk, SOCK_BROADCAST)) {
                ip_rt_put(rt);
                err = -EACCES;
                goto out;
        }

        /* Update addresses before rehashing */
        inet->inet_daddr = fl4->daddr;
        inet->inet_dport = usin->sin_port;
        if (!inet->inet_saddr)
                inet->inet_saddr = fl4->saddr;
        if (!inet->inet_rcv_saddr) {
                inet->inet_rcv_saddr = fl4->saddr;
                if (sk->sk_prot->rehash)
                        sk->sk_prot->rehash(sk);
        }
        reuseport_has_conns_set(sk);
        sk->sk_state = TCP_ESTABLISHED;
        sk_set_txhash(sk);
        atomic_set(&inet->inet_id, get_random_u16());

        sk_dst_set(sk, &rt->dst);
        err = 0;
out:
        return err;
}
EXPORT_SYMBOL(__ip4_datagram_connect);

int ip4_datagram_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len)
{
        int res;

        lock_sock(sk);
        res = __ip4_datagram_connect(sk, uaddr, addr_len);
        release_sock(sk);
        return res;
}
EXPORT_SYMBOL(ip4_datagram_connect);

/* Because UDP xmit path can manipulate sk_dst_cache without holding
 * socket lock, we need to use sk_dst_set() here,
 * even if we own the socket lock.
 */
void ip4_datagram_release_cb(struct sock *sk)
{
        const struct inet_sock *inet = inet_sk(sk);
        struct dst_entry *dst;
        struct flowi4 fl4;
        struct rtable *rt;

        rcu_read_lock();

        dst = __sk_dst_get(sk);
        if (!dst || !READ_ONCE(dst->obsolete) || dst->ops->check(dst, 0)) {
                rcu_read_unlock();
                return;
        }

        inet_sk_init_flowi4(inet, &fl4);
        rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
        dst = !IS_ERR(rt) ? &rt->dst : NULL;
        sk_dst_set(sk, dst);

        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(ip4_datagram_release_cb);














































































































































    1 



















    1 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
/* SPDX-License-Identifier: GPL-2.0 */
/* zpdesc.h: zsmalloc pool memory descriptor
 *
 * Written by Alex Shi <alexs@kernel.org>
 *              Hyeonggon Yoo <42.hyeyoo@gmail.com>
 */
#ifndef __MM_ZPDESC_H__
#define __MM_ZPDESC_H__

#include <linux/migrate.h>
#include <linux/pagemap.h>

/*
 * struct zpdesc -        Memory descriptor for zsmalloc pool memory.
 * @flags:                Page flags, mostly unused by zsmalloc.
 * @lru:                Indirectly used by page migration.
 * @movable_ops:        Used by page migration.
 * @next:                Next zpdesc in a zspage in zsmalloc pool.
 * @handle:                For huge zspage in zsmalloc pool.
 * @zspage:                Points to the zspage this zpdesc is a part of.
 * @first_obj_offset:        First object offset in zsmalloc pool.
 * @_refcount:                The number of references to this zpdesc.
 *
 * This struct overlays struct page for now. Do not modify without a good
 * understanding of the issues. In particular, do not expand into the overlap
 * with memcg_data.
 *
 * Page flags used:
 * * PG_private identifies the first component page.
 * * PG_locked is used by page migration code.
 */
struct zpdesc {
        unsigned long flags;
        struct list_head lru;
        unsigned long movable_ops;
        union {
                struct zpdesc *next;
                unsigned long handle;
        };
        struct zspage *zspage;
        /*
         * Only the lower 24 bits are available for offset, limiting a page
         * to 16 MiB. The upper 8 bits are reserved for PGTY_zsmalloc.
         *
         * Do not access this field directly.
         * Instead, use {get,set}_first_obj_offset() helpers.
         */
        unsigned int first_obj_offset;
        atomic_t _refcount;
};
#define ZPDESC_MATCH(pg, zp) \
        static_assert(offsetof(struct page, pg) == offsetof(struct zpdesc, zp))

ZPDESC_MATCH(flags, flags);
ZPDESC_MATCH(lru, lru);
ZPDESC_MATCH(mapping, movable_ops);
ZPDESC_MATCH(__folio_index, next);
ZPDESC_MATCH(__folio_index, handle);
ZPDESC_MATCH(private, zspage);
ZPDESC_MATCH(page_type, first_obj_offset);
ZPDESC_MATCH(_refcount, _refcount);
#undef ZPDESC_MATCH
static_assert(sizeof(struct zpdesc) <= sizeof(struct page));

/*
 * zpdesc_page - The first struct page allocated for a zpdesc
 * @zp: The zpdesc.
 *
 * A convenience wrapper for converting zpdesc to the first struct page of the
 * underlying folio, to communicate with code not yet converted to folio or
 * struct zpdesc.
 *
 */
#define zpdesc_page(zp)                        (_Generic((zp),                        \
        const struct zpdesc *:                (const struct page *)(zp),        \
        struct zpdesc *:                (struct page *)(zp)))

/**
 * zpdesc_folio - The folio allocated for a zpdesc
 * @zp: The zpdesc.
 *
 * Zpdescs are descriptors for zsmalloc memory. The memory itself is allocated
 * as folios that contain the zsmalloc objects, and zpdesc uses specific
 * fields in the first struct page of the folio - those fields are now accessed
 * by struct zpdesc.
 *
 * It is occasionally necessary convert to back to a folio in order to
 * communicate with the rest of the mm. Please use this helper function
 * instead of casting yourself, as the implementation may change in the future.
 */
#define zpdesc_folio(zp)                (_Generic((zp),                        \
        const struct zpdesc *:                (const struct folio *)(zp),        \
        struct zpdesc *:                (struct folio *)(zp)))
/**
 * page_zpdesc - Converts from first struct page to zpdesc.
 * @p: The first (either head of compound or single) page of zpdesc.
 *
 * A temporary wrapper to convert struct page to struct zpdesc in situations
 * where we know the page is the compound head, or single order-0 page.
 *
 * Long-term ideally everything would work with struct zpdesc directly or go
 * through folio to struct zpdesc.
 *
 * Return: The zpdesc which contains this page
 */
#define page_zpdesc(p)                        (_Generic((p),                        \
        const struct page *:                (const struct zpdesc *)(p),        \
        struct page *:                        (struct zpdesc *)(p)))

static inline void zpdesc_lock(struct zpdesc *zpdesc)
{
        folio_lock(zpdesc_folio(zpdesc));
}

static inline bool zpdesc_trylock(struct zpdesc *zpdesc)
{
        return folio_trylock(zpdesc_folio(zpdesc));
}

static inline void zpdesc_unlock(struct zpdesc *zpdesc)
{
        folio_unlock(zpdesc_folio(zpdesc));
}

static inline void zpdesc_wait_locked(struct zpdesc *zpdesc)
{
        folio_wait_locked(zpdesc_folio(zpdesc));
}

static inline void zpdesc_get(struct zpdesc *zpdesc)
{
        folio_get(zpdesc_folio(zpdesc));
}

static inline void zpdesc_put(struct zpdesc *zpdesc)
{
        folio_put(zpdesc_folio(zpdesc));
}

static inline void *kmap_local_zpdesc(struct zpdesc *zpdesc)
{
        return kmap_local_page(zpdesc_page(zpdesc));
}

static inline unsigned long zpdesc_pfn(struct zpdesc *zpdesc)
{
        return page_to_pfn(zpdesc_page(zpdesc));
}

static inline struct zpdesc *pfn_zpdesc(unsigned long pfn)
{
        return page_zpdesc(pfn_to_page(pfn));
}

static inline void __zpdesc_set_movable(struct zpdesc *zpdesc)
{
        SetPageMovableOps(zpdesc_page(zpdesc));
}

static inline void __zpdesc_set_zsmalloc(struct zpdesc *zpdesc)
{
        __SetPageZsmalloc(zpdesc_page(zpdesc));
}

static inline struct zone *zpdesc_zone(struct zpdesc *zpdesc)
{
        return page_zone(zpdesc_page(zpdesc));
}

static inline bool zpdesc_is_locked(struct zpdesc *zpdesc)
{
        return folio_test_locked(zpdesc_folio(zpdesc));
}
#endif





























































































   48 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
/* SPDX-License-Identifier: GPL-2.0 */
/*
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
 */

#ifndef _ASM_X86_STACKTRACE_H
#define _ASM_X86_STACKTRACE_H

#include <linux/uaccess.h>
#include <linux/ptrace.h>

#include <asm/cpu_entry_area.h>
#include <asm/switch_to.h>

enum stack_type {
        STACK_TYPE_UNKNOWN,
        STACK_TYPE_TASK,
        STACK_TYPE_IRQ,
        STACK_TYPE_SOFTIRQ,
        STACK_TYPE_ENTRY,
        STACK_TYPE_EXCEPTION,
        STACK_TYPE_EXCEPTION_LAST = STACK_TYPE_EXCEPTION + N_EXCEPTION_STACKS-1,
};

struct stack_info {
        enum stack_type type;
        unsigned long *begin, *end, *next_sp;
};

bool in_task_stack(unsigned long *stack, struct task_struct *task,
                   struct stack_info *info);

bool in_entry_stack(unsigned long *stack, struct stack_info *info);

int get_stack_info(unsigned long *stack, struct task_struct *task,
                   struct stack_info *info, unsigned long *visit_mask);
bool get_stack_info_noinstr(unsigned long *stack, struct task_struct *task,
                            struct stack_info *info);

static __always_inline
bool get_stack_guard_info(unsigned long *stack, struct stack_info *info)
{
        /* make sure it's not in the stack proper */
        if (get_stack_info_noinstr(stack, current, info))
                return false;
        /* but if it is in the page below it, we hit a guard */
        return get_stack_info_noinstr((void *)stack + PAGE_SIZE, current, info);
}

const char *stack_type_name(enum stack_type type);

static inline bool on_stack(struct stack_info *info, void *addr, size_t len)
{
        void *begin = info->begin;
        void *end   = info->end;

        return (info->type != STACK_TYPE_UNKNOWN &&
                addr >= begin && addr < end &&
                addr + len > begin && addr + len <= end);
}

#ifdef CONFIG_X86_32
#define STACKSLOTS_PER_LINE 8
#else
#define STACKSLOTS_PER_LINE 4
#endif

#ifdef CONFIG_FRAME_POINTER
static inline unsigned long *
get_frame_pointer(struct task_struct *task, struct pt_regs *regs)
{
        if (regs)
                return (unsigned long *)regs->bp;

        if (task == current)
                return __builtin_frame_address(0);

        return &((struct inactive_task_frame *)task->thread.sp)->bp;
}
#else
static inline unsigned long *
get_frame_pointer(struct task_struct *task, struct pt_regs *regs)
{
        return NULL;
}
#endif /* CONFIG_FRAME_POINTER */

static inline unsigned long *
get_stack_pointer(struct task_struct *task, struct pt_regs *regs)
{
        if (regs)
                return (unsigned long *)regs->sp;

        if (task == current)
                return __builtin_frame_address(0);

        return (unsigned long *)task->thread.sp;
}

/* The form of the top of the frame on the stack */
struct stack_frame {
        struct stack_frame *next_frame;
        unsigned long return_address;
};

struct stack_frame_ia32 {
    u32 next_frame;
    u32 return_address;
};

void show_opcodes(struct pt_regs *regs, const char *loglvl);
void show_ip(struct pt_regs *regs, const char *loglvl);
#endif /* _ASM_X86_STACKTRACE_H */





































  355 
  355 



    7 














    5 
    5 








  261 

  260 
   31 










   10 
    1 










  251 



  251 

    2 








    2 







   33 




   33 
   33 



    2 




    6 
   33 




    5 







    5 

    5 











    7 






    7 


















   15 



   15 
   10 







    6 


   15 

    6 




   10 

    1 
    1 






   19 




   18 
   19 

    2 



    3 





   19 









































  250 
  251 






  250 











 1355 


 1349 

  549 



  251 
  251 













   30 






   30 













































    3 







    1 

    3 








    2 










    2 










   18 





   18 




   19 





   19 
   19 
   19 
   19 

    1 
   19 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
// SPDX-License-Identifier: GPL-2.0-only
/*
 * File: pn_dev.c
 *
 * Phonet network device
 *
 * Copyright (C) 2008 Nokia Corporation.
 *
 * Authors: Sakari Ailus <sakari.ailus@nokia.com>
 *          Rémi Denis-Courmont
 */

#include <linux/kernel.h>
#include <linux/net.h>
#include <linux/slab.h>
#include <linux/netdevice.h>
#include <linux/phonet.h>
#include <linux/proc_fs.h>
#include <linux/if_arp.h>
#include <net/sock.h>
#include <net/netns/generic.h>
#include <net/phonet/pn_dev.h>

struct phonet_routes {
        spinlock_t                lock;
        struct net_device __rcu        *table[64];
};

struct phonet_net {
        struct phonet_device_list pndevs;
        struct phonet_routes routes;
};

static unsigned int phonet_net_id __read_mostly;

static struct phonet_net *phonet_pernet(struct net *net)
{
        return net_generic(net, phonet_net_id);
}

struct phonet_device_list *phonet_device_list(struct net *net)
{
        struct phonet_net *pnn = phonet_pernet(net);
        return &pnn->pndevs;
}

/* Allocate new Phonet device. */
static struct phonet_device *__phonet_device_alloc(struct net_device *dev)
{
        struct phonet_device_list *pndevs = phonet_device_list(dev_net(dev));
        struct phonet_device *pnd = kmalloc_obj(*pnd, GFP_ATOMIC);
        if (pnd == NULL)
                return NULL;
        pnd->netdev = dev;
        bitmap_zero(pnd->addrs, 64);

        lockdep_assert_held(&pndevs->lock);
        list_add_rcu(&pnd->list, &pndevs->list);
        return pnd;
}

static struct phonet_device *__phonet_get(struct net_device *dev)
{
        struct phonet_device_list *pndevs = phonet_device_list(dev_net(dev));
        struct phonet_device *pnd;

        lockdep_assert_held(&pndevs->lock);

        list_for_each_entry(pnd, &pndevs->list, list) {
                if (pnd->netdev == dev)
                        return pnd;
        }
        return NULL;
}

static struct phonet_device *__phonet_get_rcu(struct net_device *dev)
{
        struct phonet_device_list *pndevs = phonet_device_list(dev_net(dev));
        struct phonet_device *pnd;

        list_for_each_entry_rcu(pnd, &pndevs->list, list) {
                if (pnd->netdev == dev)
                        return pnd;
        }
        return NULL;
}

static void phonet_device_destroy(struct net_device *dev)
{
        struct phonet_device_list *pndevs = phonet_device_list(dev_net(dev));
        struct phonet_device *pnd;

        ASSERT_RTNL();

        spin_lock(&pndevs->lock);

        pnd = __phonet_get(dev);
        if (pnd)
                list_del_rcu(&pnd->list);

        spin_unlock(&pndevs->lock);

        if (pnd) {
                struct net *net = dev_net(dev);
                u32 ifindex = dev->ifindex;
                u8 addr;

                for_each_set_bit(addr, pnd->addrs, 64)
                        phonet_address_notify(net, RTM_DELADDR, ifindex, addr);

                kfree_rcu(pnd, rcu);
        }
}

struct net_device *phonet_device_get(struct net *net)
{
        struct phonet_device_list *pndevs = phonet_device_list(net);
        struct phonet_device *pnd;
        struct net_device *dev = NULL;

        rcu_read_lock();
        list_for_each_entry_rcu(pnd, &pndevs->list, list) {
                dev = pnd->netdev;
                BUG_ON(!dev);

                if ((dev->reg_state == NETREG_REGISTERED) &&
                        ((pnd->netdev->flags & IFF_UP)) == IFF_UP)
                        break;
                dev = NULL;
        }
        dev_hold(dev);
        rcu_read_unlock();
        return dev;
}

int phonet_address_add(struct net_device *dev, u8 addr)
{
        struct phonet_device_list *pndevs = phonet_device_list(dev_net(dev));
        struct phonet_device *pnd;
        int err = 0;

        spin_lock(&pndevs->lock);

        /* Find or create Phonet-specific device data */
        pnd = __phonet_get(dev);
        if (pnd == NULL)
                pnd = __phonet_device_alloc(dev);
        if (unlikely(pnd == NULL))
                err = -ENOMEM;
        else if (test_and_set_bit(addr >> 2, pnd->addrs))
                err = -EEXIST;

        spin_unlock(&pndevs->lock);

        return err;
}

int phonet_address_del(struct net_device *dev, u8 addr)
{
        struct phonet_device_list *pndevs = phonet_device_list(dev_net(dev));
        struct phonet_device *pnd;
        int err = 0;

        spin_lock(&pndevs->lock);

        pnd = __phonet_get(dev);
        if (!pnd || !test_and_clear_bit(addr >> 2, pnd->addrs)) {
                err = -EADDRNOTAVAIL;
                pnd = NULL;
        } else if (bitmap_empty(pnd->addrs, 64))
                list_del_rcu(&pnd->list);
        else
                pnd = NULL;

        spin_unlock(&pndevs->lock);

        if (pnd)
                kfree_rcu(pnd, rcu);

        return err;
}

/* Gets a source address toward a destination, through a interface. */
u8 phonet_address_get(struct net_device *dev, u8 daddr)
{
        struct phonet_device *pnd;
        u8 saddr;

        rcu_read_lock();
        pnd = __phonet_get_rcu(dev);
        if (pnd) {
                BUG_ON(bitmap_empty(pnd->addrs, 64));

                /* Use same source address as destination, if possible */
                if (test_bit(daddr >> 2, pnd->addrs))
                        saddr = daddr;
                else
                        saddr = find_first_bit(pnd->addrs, 64) << 2;
        } else
                saddr = PN_NO_ADDR;
        rcu_read_unlock();

        if (saddr == PN_NO_ADDR) {
                /* Fallback to another device */
                struct net_device *def_dev;

                def_dev = phonet_device_get(dev_net(dev));
                if (def_dev) {
                        if (def_dev != dev)
                                saddr = phonet_address_get(def_dev, daddr);
                        dev_put(def_dev);
                }
        }
        return saddr;
}

int phonet_address_lookup(struct net *net, u8 addr)
{
        struct phonet_device_list *pndevs = phonet_device_list(net);
        struct phonet_device *pnd;
        int err = -EADDRNOTAVAIL;

        rcu_read_lock();
        list_for_each_entry_rcu(pnd, &pndevs->list, list) {
                /* Don't allow unregistering devices! */
                if ((pnd->netdev->reg_state != NETREG_REGISTERED) ||
                                ((pnd->netdev->flags & IFF_UP)) != IFF_UP)
                        continue;

                if (test_bit(addr >> 2, pnd->addrs)) {
                        err = 0;
                        goto found;
                }
        }
found:
        rcu_read_unlock();
        return err;
}

/* automatically configure a Phonet device, if supported */
static int phonet_device_autoconf(struct net_device *dev)
{
        struct if_phonet_req req;
        int ret;

        if (!dev->netdev_ops->ndo_siocdevprivate)
                return -EOPNOTSUPP;

        ret = dev->netdev_ops->ndo_siocdevprivate(dev, (struct ifreq *)&req,
                                                  NULL, SIOCPNGAUTOCONF);
        if (ret < 0)
                return ret;

        ASSERT_RTNL();
        ret = phonet_address_add(dev, req.ifr_phonet_autoconf.device);
        if (ret)
                return ret;

        phonet_address_notify(dev_net(dev), RTM_NEWADDR, dev->ifindex,
                              req.ifr_phonet_autoconf.device);
        return 0;
}

static void phonet_route_autodel(struct net_device *dev)
{
        struct net *net = dev_net(dev);
        DECLARE_BITMAP(deleted, 64);
        u32 ifindex = dev->ifindex;
        struct phonet_net *pnn;
        unsigned int i;

        pnn = phonet_pernet(net);

        /* Remove left-over Phonet routes */
        bitmap_zero(deleted, 64);

        spin_lock(&pnn->routes.lock);
        for (i = 0; i < 64; i++) {
                if (rcu_access_pointer(pnn->routes.table[i]) == dev) {
                        RCU_INIT_POINTER(pnn->routes.table[i], NULL);
                        set_bit(i, deleted);
                }
        }
        spin_unlock(&pnn->routes.lock);

        if (bitmap_empty(deleted, 64))
                return; /* short-circuit RCU */
        synchronize_rcu();
        for_each_set_bit(i, deleted, 64) {
                rtm_phonet_notify(net, RTM_DELROUTE, ifindex, i);
                dev_put(dev);
        }
}

/* notify Phonet of device events */
static int phonet_device_notify(struct notifier_block *me, unsigned long what,
                                void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);

        switch (what) {
        case NETDEV_REGISTER:
                if (dev->type == ARPHRD_PHONET)
                        phonet_device_autoconf(dev);
                break;
        case NETDEV_UNREGISTER:
                phonet_device_destroy(dev);
                phonet_route_autodel(dev);
                break;
        }
        return 0;

}

static struct notifier_block phonet_device_notifier = {
        .notifier_call = phonet_device_notify,
        .priority = 0,
};

/* Per-namespace Phonet devices handling */
static int __net_init phonet_init_net(struct net *net)
{
        struct phonet_net *pnn = phonet_pernet(net);

        if (!proc_create_net("phonet", 0, net->proc_net, &pn_sock_seq_ops,
                        sizeof(struct seq_net_private)))
                return -ENOMEM;

        INIT_LIST_HEAD(&pnn->pndevs.list);
        spin_lock_init(&pnn->pndevs.lock);
        spin_lock_init(&pnn->routes.lock);
        return 0;
}

static void __net_exit phonet_exit_net(struct net *net)
{
        struct phonet_net *pnn = phonet_pernet(net);

        remove_proc_entry("phonet", net->proc_net);
        WARN_ON_ONCE(!list_empty(&pnn->pndevs.list));
}

static struct pernet_operations phonet_net_ops = {
        .init = phonet_init_net,
        .exit = phonet_exit_net,
        .id   = &phonet_net_id,
        .size = sizeof(struct phonet_net),
};

/* Initialize Phonet devices list */
int __init phonet_device_init(void)
{
        int err = register_pernet_subsys(&phonet_net_ops);
        if (err)
                return err;

        proc_create_net("pnresource", 0, init_net.proc_net, &pn_res_seq_ops,
                        sizeof(struct seq_net_private));
        register_netdevice_notifier(&phonet_device_notifier);
        err = phonet_netlink_register();
        if (err)
                phonet_device_exit();
        return err;
}

void phonet_device_exit(void)
{
        rtnl_unregister_all(PF_PHONET);
        unregister_netdevice_notifier(&phonet_device_notifier);
        unregister_pernet_subsys(&phonet_net_ops);
        remove_proc_entry("pnresource", init_net.proc_net);
}

int phonet_route_add(struct net_device *dev, u8 daddr)
{
        struct phonet_net *pnn = phonet_pernet(dev_net(dev));
        struct phonet_routes *routes = &pnn->routes;
        int err = -EEXIST;

        daddr = daddr >> 2;

        spin_lock(&routes->lock);
        if (routes->table[daddr] == NULL) {
                rcu_assign_pointer(routes->table[daddr], dev);
                dev_hold(dev);
                err = 0;
        }
        spin_unlock(&routes->lock);

        return err;
}

int phonet_route_del(struct net_device *dev, u8 daddr)
{
        struct phonet_net *pnn = phonet_pernet(dev_net(dev));
        struct phonet_routes *routes = &pnn->routes;

        daddr = daddr >> 2;

        spin_lock(&routes->lock);
        if (rcu_access_pointer(routes->table[daddr]) == dev)
                RCU_INIT_POINTER(routes->table[daddr], NULL);
        else
                dev = NULL;
        spin_unlock(&routes->lock);

        if (!dev)
                return -ENOENT;

        /* Note : our caller must call synchronize_rcu() and dev_put(dev) */

        return 0;
}

struct net_device *phonet_route_get_rcu(struct net *net, u8 daddr)
{
        struct phonet_net *pnn = phonet_pernet(net);
        struct phonet_routes *routes = &pnn->routes;
        struct net_device *dev;

        daddr >>= 2;
        dev = rcu_dereference(routes->table[daddr]);
        return dev;
}

struct net_device *phonet_route_output(struct net *net, u8 daddr)
{
        struct phonet_net *pnn = phonet_pernet(net);
        struct phonet_routes *routes = &pnn->routes;
        struct net_device *dev;

        daddr >>= 2;
        rcu_read_lock();
        dev = rcu_dereference(routes->table[daddr]);
        dev_hold(dev);
        rcu_read_unlock();

        if (!dev)
                dev = phonet_device_get(net); /* Default route */
        return dev;
}






































 2438 
 2443 





 1879 

 1881 





 2439 



 1454 






    2 





 2195 

  854 















































  715 

  716 





















  715 





































  715 



































  461 





    4 
    9 





  716 





  467 



















  457 









  456 






















  716 
  715 
  714 

  716 






  716 
  716 




  716 


  712 
   17 






   68 





    8 






   54 
   16 


  455 


  187 








  650 
   69 
  259 
  457 




  715 
  713 







  714 







  715 






  716 


  715 








 1490 


























 1491 


  286 

 1207 

 1493 

 1384 
 1036 












 1441 













 1309 



    1 



  133 


   27 



  102 
    8 









    8 
    8 














   16 
 1416 












    5 





    5 




    5 




    5 




    5 




    5 








    5 





    5 




    5 




    5 











































































































 2493 



    1 







 2494 










    7 





    1 


    6 











    5 

















    3 


    2 













  509 






  506 
  506 
  507 







  520 







    1 
    2 
    2 


    2 



    1 


  508 

    3 
    1 









  508 
    3 







    1 


  510 

    2 
  506 







 2519 






 2517 



   18 





   11 




   69 
    3 









 1417 




 1397 







   20 

   15 

















   30 
    1 



 1212 


 1187 



    1 




    1 







    1 





    1 





    1 







    1 





    1 















   17 


    2 












   18 







  780 
   17 

  761 





















 1233 















    1 
    5 

























    4 

 1190 
    3 



   27 





    1 







    3 























 1183 




   34 



    1 












   34 

   34 











 1209 

    7 







  958 

    2 



   85 





    1 



  169 








  513 
  699 
   99 
 1110 


    8 


 1060 
  143 






  829 






  374 





    1 

  447 
  338 










  684 














  755 

   22 


   15 
  764 


  800 
    6 


    5 
  801 





 1017 




  997 


    1 





    1 


    6 


    4 


    3 







    4 









    3 




  540 
  531 





    6 
    4 




 1036 














    2 



    1 


    1 



  527 

  196 

  311 





    3 

 1003 

    2 
    2 
   22 



    1 



   20 















 1021 


    2 




    2 










    9 



  888 


  113 




    2 










  119 





  119 










  102 
  898 


  716 

  286 












    5 
  527 



  106 
  435 

  547 
   86 







    2 

    1 




    1 









  237 












    3 







    3 








    2 


  228 

    1 

    1 










  213 


    1 




    1 
  122 










   22 








    3 


    4 








    3 

    1 


    1 


    1 


    1 









    1 

    2 





    5 




















    7 
    7 






    5 










    5 










  331 







    2 
    2 
    3 



    1 



  318 

    1 
    5 






  318 




    1 









   14 
    2 


    3 


    9 


    8 












































   25 
















    4 









    4 






    2 



    1 

   14 

















    2 



    2 

    1 



    1 

    2 









    1 








    3 










































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2017 Facebook
 */
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/vmalloc.h>
#include <linux/etherdevice.h>
#include <linux/filter.h>
#include <linux/rcupdate_trace.h>
#include <linux/sched/signal.h>
#include <net/bpf_sk_storage.h>
#include <net/hotdata.h>
#include <net/sock.h>
#include <net/tcp.h>
#include <net/net_namespace.h>
#include <net/page_pool/helpers.h>
#include <linux/error-injection.h>
#include <linux/smp.h>
#include <linux/sock_diag.h>
#include <linux/netfilter.h>
#include <net/netdev_rx_queue.h>
#include <net/xdp.h>
#include <net/netfilter/nf_bpf_link.h>

#define CREATE_TRACE_POINTS
#include <trace/events/bpf_test_run.h>

struct bpf_test_timer {
        u32 i;
        u64 time_start, time_spent;
};

static void bpf_test_timer_enter(struct bpf_test_timer *t)
        __acquires(rcu)
{
        rcu_read_lock_dont_migrate();
        t->time_start = ktime_get_ns();
}

static void bpf_test_timer_leave(struct bpf_test_timer *t)
        __releases(rcu)
{
        t->time_start = 0;
        rcu_read_unlock_migrate();
}

static bool bpf_test_timer_continue(struct bpf_test_timer *t, int iterations,
                                    u32 repeat, int *err, u32 *duration)
        __must_hold(rcu)
{
        t->i += iterations;
        if (t->i >= repeat) {
                /* We're done. */
                t->time_spent += ktime_get_ns() - t->time_start;
                do_div(t->time_spent, t->i);
                *duration = t->time_spent > U32_MAX ? U32_MAX : (u32)t->time_spent;
                *err = 0;
                goto reset;
        }

        if (signal_pending(current)) {
                /* During iteration: we've been cancelled, abort. */
                *err = -EINTR;
                goto reset;
        }

        if (need_resched()) {
                /* During iteration: we need to reschedule between runs. */
                t->time_spent += ktime_get_ns() - t->time_start;
                bpf_test_timer_leave(t);
                cond_resched();
                bpf_test_timer_enter(t);
        }

        /* Do another round. */
        return true;

reset:
        t->i = 0;
        return false;
}

/* We put this struct at the head of each page with a context and frame
 * initialised when the page is allocated, so we don't have to do this on each
 * repetition of the test run.
 */
struct xdp_page_head {
        struct xdp_buff orig_ctx;
        struct xdp_buff ctx;
        union {
                /* ::data_hard_start starts here */
                DECLARE_FLEX_ARRAY(struct xdp_frame, frame);
                DECLARE_FLEX_ARRAY(u8, data);
        };
};

struct xdp_test_data {
        struct xdp_buff *orig_ctx;
        struct xdp_rxq_info rxq;
        struct net_device *dev;
        struct page_pool *pp;
        struct xdp_frame **frames;
        struct sk_buff **skbs;
        struct xdp_mem_info mem;
        u32 batch_size;
        u32 frame_cnt;
};

/* tools/testing/selftests/bpf/prog_tests/xdp_do_redirect.c:%MAX_PKT_SIZE
 * must be updated accordingly this gets changed, otherwise BPF selftests
 * will fail.
 */
#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
#define TEST_XDP_MAX_BATCH 256

static void xdp_test_run_init_page(netmem_ref netmem, void *arg)
{
        struct xdp_page_head *head =
                phys_to_virt(page_to_phys(netmem_to_page(netmem)));
        struct xdp_buff *new_ctx, *orig_ctx;
        u32 headroom = XDP_PACKET_HEADROOM;
        struct xdp_test_data *xdp = arg;
        size_t frm_len, meta_len;
        struct xdp_frame *frm;
        void *data;

        orig_ctx = xdp->orig_ctx;
        frm_len = orig_ctx->data_end - orig_ctx->data_meta;
        meta_len = orig_ctx->data - orig_ctx->data_meta;
        headroom -= meta_len;

        new_ctx = &head->ctx;
        frm = head->frame;
        data = head->data;
        memcpy(data + headroom, orig_ctx->data_meta, frm_len);

        xdp_init_buff(new_ctx, TEST_XDP_FRAME_SIZE, &xdp->rxq);
        xdp_prepare_buff(new_ctx, data, headroom, frm_len, true);
        new_ctx->data = new_ctx->data_meta + meta_len;

        xdp_update_frame_from_buff(new_ctx, frm);
        frm->mem_type = new_ctx->rxq->mem.type;

        memcpy(&head->orig_ctx, new_ctx, sizeof(head->orig_ctx));
}

static int xdp_test_run_setup(struct xdp_test_data *xdp, struct xdp_buff *orig_ctx)
{
        struct page_pool *pp;
        int err = -ENOMEM;
        struct page_pool_params pp_params = {
                .order = 0,
                .flags = 0,
                .pool_size = xdp->batch_size,
                .nid = NUMA_NO_NODE,
                .init_callback = xdp_test_run_init_page,
                .init_arg = xdp,
        };

        xdp->frames = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);
        if (!xdp->frames)
                return -ENOMEM;

        xdp->skbs = kvmalloc_array(xdp->batch_size, sizeof(void *), GFP_KERNEL);
        if (!xdp->skbs)
                goto err_skbs;

        pp = page_pool_create(&pp_params);
        if (IS_ERR(pp)) {
                err = PTR_ERR(pp);
                goto err_pp;
        }

        /* will copy 'mem.id' into pp->xdp_mem_id */
        err = xdp_reg_mem_model(&xdp->mem, MEM_TYPE_PAGE_POOL, pp);
        if (err)
                goto err_mmodel;

        xdp->pp = pp;

        /* We create a 'fake' RXQ referencing the original dev, but with an
         * xdp_mem_info pointing to our page_pool
         */
        xdp_rxq_info_reg(&xdp->rxq, orig_ctx->rxq->dev, 0, 0);
        xdp->rxq.mem.type = MEM_TYPE_PAGE_POOL;
        xdp->rxq.mem.id = pp->xdp_mem_id;
        xdp->dev = orig_ctx->rxq->dev;
        xdp->orig_ctx = orig_ctx;

        return 0;

err_mmodel:
        page_pool_destroy(pp);
err_pp:
        kvfree(xdp->skbs);
err_skbs:
        kvfree(xdp->frames);
        return err;
}

static void xdp_test_run_teardown(struct xdp_test_data *xdp)
{
        xdp_unreg_mem_model(&xdp->mem);
        page_pool_destroy(xdp->pp);
        kfree(xdp->frames);
        kfree(xdp->skbs);
}

static bool frame_was_changed(const struct xdp_page_head *head)
{
        /* xdp_scrub_frame() zeroes the data pointer, flags is the last field,
         * i.e. has the highest chances to be overwritten. If those two are
         * untouched, it's most likely safe to skip the context reset.
         */
        return head->frame->data != head->orig_ctx.data ||
               head->frame->flags != head->orig_ctx.flags;
}

static bool ctx_was_changed(struct xdp_page_head *head)
{
        return head->orig_ctx.data != head->ctx.data ||
                head->orig_ctx.data_meta != head->ctx.data_meta ||
                head->orig_ctx.data_end != head->ctx.data_end;
}

static void reset_ctx(struct xdp_page_head *head)
{
        if (likely(!frame_was_changed(head) && !ctx_was_changed(head)))
                return;

        head->ctx.data = head->orig_ctx.data;
        head->ctx.data_meta = head->orig_ctx.data_meta;
        head->ctx.data_end = head->orig_ctx.data_end;
        xdp_update_frame_from_buff(&head->ctx, head->frame);
        head->frame->mem_type = head->orig_ctx.rxq->mem.type;
}

static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
                           struct sk_buff **skbs,
                           struct net_device *dev)
{
        gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
        int i, n;
        LIST_HEAD(list);

        n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,
                                  (void **)skbs);
        if (unlikely(n == 0)) {
                for (i = 0; i < nframes; i++)
                        xdp_return_frame(frames[i]);
                return -ENOMEM;
        }

        for (i = 0; i < nframes; i++) {
                struct xdp_frame *xdpf = frames[i];
                struct sk_buff *skb = skbs[i];

                skb = __xdp_build_skb_from_frame(xdpf, skb, dev);
                if (!skb) {
                        xdp_return_frame(xdpf);
                        continue;
                }

                list_add_tail(&skb->list, &list);
        }
        netif_receive_skb_list(&list);

        return 0;
}

static int xdp_test_run_batch(struct xdp_test_data *xdp, struct bpf_prog *prog,
                              u32 repeat)
{
        struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
        int err = 0, act, ret, i, nframes = 0, batch_sz;
        struct xdp_frame **frames = xdp->frames;
        struct bpf_redirect_info *ri;
        struct xdp_page_head *head;
        struct xdp_frame *frm;
        bool redirect = false;
        struct xdp_buff *ctx;
        struct page *page;

        batch_sz = min_t(u32, repeat, xdp->batch_size);

        local_bh_disable();
        bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
        ri = bpf_net_ctx_get_ri();
        xdp_set_return_frame_no_direct();

        for (i = 0; i < batch_sz; i++) {
                page = page_pool_dev_alloc_pages(xdp->pp);
                if (!page) {
                        err = -ENOMEM;
                        goto out;
                }

                head = phys_to_virt(page_to_phys(page));
                reset_ctx(head);
                ctx = &head->ctx;
                frm = head->frame;
                xdp->frame_cnt++;

                act = bpf_prog_run_xdp(prog, ctx);

                /* if program changed pkt bounds we need to update the xdp_frame */
                if (unlikely(ctx_was_changed(head))) {
                        ret = xdp_update_frame_from_buff(ctx, frm);
                        if (ret) {
                                xdp_return_buff(ctx);
                                continue;
                        }
                }

                switch (act) {
                case XDP_TX:
                        /* we can't do a real XDP_TX since we're not in the
                         * driver, so turn it into a REDIRECT back to the same
                         * index
                         */
                        ri->tgt_index = xdp->dev->ifindex;
                        ri->map_id = INT_MAX;
                        ri->map_type = BPF_MAP_TYPE_UNSPEC;
                        fallthrough;
                case XDP_REDIRECT:
                        redirect = true;
                        ret = xdp_do_redirect_frame(xdp->dev, ctx, frm, prog);
                        if (ret)
                                xdp_return_buff(ctx);
                        break;
                case XDP_PASS:
                        frames[nframes++] = frm;
                        break;
                default:
                        bpf_warn_invalid_xdp_action(NULL, prog, act);
                        fallthrough;
                case XDP_DROP:
                        xdp_return_buff(ctx);
                        break;
                }
        }

out:
        if (redirect)
                xdp_do_flush();
        if (nframes) {
                ret = xdp_recv_frames(frames, nframes, xdp->skbs, xdp->dev);
                if (ret)
                        err = ret;
        }

        xdp_clear_return_frame_no_direct();
        bpf_net_ctx_clear(bpf_net_ctx);
        local_bh_enable();
        return err;
}

static int bpf_test_run_xdp_live(struct bpf_prog *prog, struct xdp_buff *ctx,
                                 u32 repeat, u32 batch_size, u32 *time)

{
        struct xdp_test_data xdp = { .batch_size = batch_size };
        struct bpf_test_timer t = {};
        int ret;

        if (!repeat)
                repeat = 1;

        ret = xdp_test_run_setup(&xdp, ctx);
        if (ret)
                return ret;

        bpf_test_timer_enter(&t);
        do {
                xdp.frame_cnt = 0;
                ret = xdp_test_run_batch(&xdp, prog, repeat - t.i);
                if (unlikely(ret < 0))
                        break;
        } while (bpf_test_timer_continue(&t, xdp.frame_cnt, repeat, &ret, time));
        bpf_test_timer_leave(&t);

        xdp_test_run_teardown(&xdp);
        return ret;
}

static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat,
                        u32 *retval, u32 *time, bool xdp)
{
        struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
        struct bpf_prog_array_item item = {.prog = prog};
        struct bpf_run_ctx *old_ctx;
        struct bpf_cg_run_ctx run_ctx;
        struct bpf_test_timer t = {};
        enum bpf_cgroup_storage_type stype;
        int ret;

        for_each_cgroup_storage_type(stype) {
                item.cgroup_storage[stype] = bpf_cgroup_storage_alloc(prog, stype);
                if (IS_ERR(item.cgroup_storage[stype])) {
                        item.cgroup_storage[stype] = NULL;
                        for_each_cgroup_storage_type(stype)
                                bpf_cgroup_storage_free(item.cgroup_storage[stype]);
                        return -ENOMEM;
                }
        }

        if (!repeat)
                repeat = 1;

        bpf_test_timer_enter(&t);
        old_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
        do {
                run_ctx.prog_item = &item;
                local_bh_disable();
                bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);

                if (xdp)
                        *retval = bpf_prog_run_xdp(prog, ctx);
                else
                        *retval = bpf_prog_run(prog, ctx);

                bpf_net_ctx_clear(bpf_net_ctx);
                local_bh_enable();
        } while (bpf_test_timer_continue(&t, 1, repeat, &ret, time));
        bpf_reset_run_ctx(old_ctx);
        bpf_test_timer_leave(&t);

        for_each_cgroup_storage_type(stype)
                bpf_cgroup_storage_free(item.cgroup_storage[stype]);

        return ret;
}

static int bpf_test_finish(const union bpf_attr *kattr,
                           union bpf_attr __user *uattr, const void *data,
                           struct skb_shared_info *sinfo, u32 size, u32 frag_size,
                           u32 retval, u32 duration)
{
        void __user *data_out = u64_to_user_ptr(kattr->test.data_out);
        int err = -EFAULT;
        u32 copy_size = size;

        /* Clamp copy if the user has provided a size hint, but copy the full
         * buffer if not to retain old behaviour.
         */
        if (kattr->test.data_size_out &&
            copy_size > kattr->test.data_size_out) {
                copy_size = kattr->test.data_size_out;
                err = -ENOSPC;
        }

        if (data_out) {
                int len = sinfo ? copy_size - frag_size : copy_size;

                if (len < 0) {
                        err = -ENOSPC;
                        goto out;
                }

                if (copy_to_user(data_out, data, len))
                        goto out;

                if (sinfo) {
                        int i, offset = len;
                        u32 data_len;

                        for (i = 0; i < sinfo->nr_frags; i++) {
                                skb_frag_t *frag = &sinfo->frags[i];

                                if (offset >= copy_size) {
                                        err = -ENOSPC;
                                        break;
                                }

                                data_len = min_t(u32, copy_size - offset,
                                                 skb_frag_size(frag));

                                if (copy_to_user(data_out + offset,
                                                 skb_frag_address(frag),
                                                 data_len))
                                        goto out;

                                offset += data_len;
                        }
                }
        }

        if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size)))
                goto out;
        if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
                goto out;
        if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration)))
                goto out;
        if (err != -ENOSPC)
                err = 0;
out:
        trace_bpf_test_finish(&err);
        return err;
}

/* Integer types of various sizes and pointer combinations cover variety of
 * architecture dependent calling conventions. 7+ can be supported in the
 * future.
 */
__bpf_kfunc_start_defs();

__bpf_kfunc int bpf_fentry_test1(int a)
{
        return a + 1;
}
EXPORT_SYMBOL_GPL(bpf_fentry_test1);

noinline int bpf_fentry_test2(int a, u64 b)
{
        return a + b;
}

noinline int bpf_fentry_test3(char a, int b, u64 c)
{
        return a + b + c;
}

noinline int bpf_fentry_test4(void *a, char b, int c, u64 d)
{
        return (long)a + b + c + d;
}

noinline int bpf_fentry_test5(u64 a, void *b, short c, int d, u64 e)
{
        return a + (long)b + c + d + e;
}

noinline int bpf_fentry_test6(u64 a, void *b, short c, int d, void *e, u64 f)
{
        return a + (long)b + c + d + (long)e + f;
}

struct bpf_fentry_test_t {
        struct bpf_fentry_test_t *a;
};

noinline int bpf_fentry_test7(struct bpf_fentry_test_t *arg)
{
        asm volatile ("" : "+r"(arg));
        return (long)arg;
}

noinline int bpf_fentry_test8(struct bpf_fentry_test_t *arg)
{
        return (long)arg->a;
}

__bpf_kfunc u32 bpf_fentry_test9(u32 *a)
{
        return *a;
}

noinline int bpf_fentry_test10(const void *a)
{
        return (long)a;
}

noinline void bpf_fentry_test_sinfo(struct skb_shared_info *sinfo)
{
}

noinline void bpf_fentry_test_ppvoid(void **pp)
{
}

noinline void bpf_fentry_test_pppvoid(void ***ppp)
{
}

noinline void bpf_fentry_test_ppfile(struct file **ppf)
{
}

noinline struct file **bpf_fexit_test_ret_ppfile(void)
{
        return (struct file **)NULL;
}

__bpf_kfunc int bpf_modify_return_test(int a, int *b)
{
        *b += 1;
        return a + *b;
}

__bpf_kfunc int bpf_modify_return_test2(int a, int *b, short c, int d,
                                        void *e, char f, int g)
{
        *b += 1;
        return a + *b + c + d + (long)e + f + g;
}

__bpf_kfunc int bpf_modify_return_test_tp(int nonce)
{
        trace_bpf_trigger_tp(nonce);

        return nonce;
}

noinline int bpf_fentry_shadow_test(int a)
{
        return a + 1;
}

struct prog_test_member1 {
        int a;
};

struct prog_test_member {
        struct prog_test_member1 m;
        int c;
};

struct prog_test_ref_kfunc {
        int a;
        int b;
        struct prog_test_member memb;
        struct prog_test_ref_kfunc *next;
        refcount_t cnt;
};

__bpf_kfunc void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p)
{
        refcount_dec(&p->cnt);
}

__bpf_kfunc void bpf_kfunc_call_test_release_dtor(void *p)
{
        bpf_kfunc_call_test_release(p);
}
CFI_NOSEAL(bpf_kfunc_call_test_release_dtor);

__bpf_kfunc void bpf_kfunc_call_memb_release(struct prog_test_member *p)
{
}

__bpf_kfunc void bpf_kfunc_call_memb_release_dtor(void *p)
{
}
CFI_NOSEAL(bpf_kfunc_call_memb_release_dtor);

__bpf_kfunc_end_defs();

BTF_KFUNCS_START(bpf_test_modify_return_ids)
BTF_ID_FLAGS(func, bpf_modify_return_test)
BTF_ID_FLAGS(func, bpf_modify_return_test2)
BTF_ID_FLAGS(func, bpf_modify_return_test_tp)
BTF_ID_FLAGS(func, bpf_fentry_test1, KF_SLEEPABLE)
BTF_KFUNCS_END(bpf_test_modify_return_ids)

static const struct btf_kfunc_id_set bpf_test_modify_return_set = {
        .owner = THIS_MODULE,
        .set   = &bpf_test_modify_return_ids,
};

BTF_KFUNCS_START(test_sk_check_kfunc_ids)
BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE)
BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE)
BTF_KFUNCS_END(test_sk_check_kfunc_ids)

static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
                           u32 size, u32 headroom, u32 tailroom)
{
        void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
        void *data;

        if (user_size > PAGE_SIZE - headroom - tailroom)
                return ERR_PTR(-EINVAL);

        size = SKB_DATA_ALIGN(size);
        data = kzalloc(size + headroom + tailroom, GFP_USER);
        if (!data)
                return ERR_PTR(-ENOMEM);

        if (copy_from_user(data + headroom, data_in, user_size)) {
                kfree(data);
                return ERR_PTR(-EFAULT);
        }

        return data;
}

int bpf_prog_test_run_tracing(struct bpf_prog *prog,
                              const union bpf_attr *kattr,
                              union bpf_attr __user *uattr)
{
        struct bpf_fentry_test_t arg = {};
        u16 side_effect = 0, ret = 0;
        int b = 2, err = -EFAULT;
        u32 retval = 0;

        if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
                return -EINVAL;

        switch (prog->expected_attach_type) {
        case BPF_TRACE_FENTRY:
        case BPF_TRACE_FEXIT:
        case BPF_TRACE_FSESSION:
                if (bpf_fentry_test1(1) != 2 ||
                    bpf_fentry_test2(2, 3) != 5 ||
                    bpf_fentry_test3(4, 5, 6) != 15 ||
                    bpf_fentry_test4((void *)7, 8, 9, 10) != 34 ||
                    bpf_fentry_test5(11, (void *)12, 13, 14, 15) != 65 ||
                    bpf_fentry_test6(16, (void *)17, 18, 19, (void *)20, 21) != 111 ||
                    bpf_fentry_test7((struct bpf_fentry_test_t *)0) != 0 ||
                    bpf_fentry_test8(&arg) != 0 ||
                    bpf_fentry_test9(&retval) != 0 ||
                    bpf_fentry_test10((void *)0) != 0)
                        goto out;
                break;
        case BPF_MODIFY_RETURN:
                ret = bpf_modify_return_test(1, &b);
                if (b != 2)
                        side_effect++;
                b = 2;
                ret += bpf_modify_return_test2(1, &b, 3, 4, (void *)5, 6, 7);
                if (b != 2)
                        side_effect++;
                break;
        default:
                goto out;
        }

        retval = ((u32)side_effect << 16) | ret;
        if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval)))
                goto out;

        err = 0;
out:
        trace_bpf_test_finish(&err);
        return err;
}

struct bpf_raw_tp_test_run_info {
        struct bpf_prog *prog;
        void *ctx;
        u32 retval;
};

static void
__bpf_prog_test_run_raw_tp(void *data)
{
        struct bpf_raw_tp_test_run_info *info = data;
        struct bpf_trace_run_ctx run_ctx = {};
        struct bpf_run_ctx *old_run_ctx;

        old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);

        rcu_read_lock();
        info->retval = bpf_prog_run(info->prog, info->ctx);
        rcu_read_unlock();

        bpf_reset_run_ctx(old_run_ctx);
}

int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
                             const union bpf_attr *kattr,
                             union bpf_attr __user *uattr)
{
        void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
        __u32 ctx_size_in = kattr->test.ctx_size_in;
        struct bpf_raw_tp_test_run_info info;
        int cpu = kattr->test.cpu, err = 0;
        int current_cpu;

        /* doesn't support data_in/out, ctx_out, duration, or repeat */
        if (kattr->test.data_in || kattr->test.data_out ||
            kattr->test.ctx_out || kattr->test.duration ||
            kattr->test.repeat || kattr->test.batch_size)
                return -EINVAL;

        if (ctx_size_in < prog->aux->max_ctx_offset ||
            ctx_size_in > MAX_BPF_FUNC_ARGS * sizeof(u64))
                return -EINVAL;

        if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
                return -EINVAL;

        if (ctx_size_in) {
                info.ctx = memdup_user(ctx_in, ctx_size_in);
                if (IS_ERR(info.ctx))
                        return PTR_ERR(info.ctx);
        } else {
                info.ctx = NULL;
        }

        info.prog = prog;

        current_cpu = get_cpu();
        if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
            cpu == current_cpu) {
                __bpf_prog_test_run_raw_tp(&info);
        } else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
                /* smp_call_function_single() also checks cpu_online()
                 * after csd_lock(). However, since cpu is from user
                 * space, let's do an extra quick check to filter out
                 * invalid value before smp_call_function_single().
                 */
                err = -ENXIO;
        } else {
                err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
                                               &info, 1);
        }
        put_cpu();

        if (!err &&
            copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
                err = -EFAULT;

        kfree(info.ctx);
        return err;
}

static void *bpf_ctx_init(const union bpf_attr *kattr, u32 max_size)
{
        void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
        void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
        u32 size = kattr->test.ctx_size_in;
        void *data;
        int err;

        if (!data_in && !data_out)
                return NULL;

        data = kzalloc(max_size, GFP_USER);
        if (!data)
                return ERR_PTR(-ENOMEM);

        if (data_in) {
                err = bpf_check_uarg_tail_zero(USER_BPFPTR(data_in), max_size, size);
                if (err) {
                        kfree(data);
                        return ERR_PTR(err);
                }

                size = min_t(u32, max_size, size);
                if (copy_from_user(data, data_in, size)) {
                        kfree(data);
                        return ERR_PTR(-EFAULT);
                }
        }
        return data;
}

static int bpf_ctx_finish(const union bpf_attr *kattr,
                          union bpf_attr __user *uattr, const void *data,
                          u32 size)
{
        void __user *data_out = u64_to_user_ptr(kattr->test.ctx_out);
        int err = -EFAULT;
        u32 copy_size = size;

        if (!data || !data_out)
                return 0;

        if (copy_size > kattr->test.ctx_size_out) {
                copy_size = kattr->test.ctx_size_out;
                err = -ENOSPC;
        }

        if (copy_to_user(data_out, data, copy_size))
                goto out;
        if (copy_to_user(&uattr->test.ctx_size_out, &size, sizeof(size)))
                goto out;
        if (err != -ENOSPC)
                err = 0;
out:
        return err;
}

/**
 * range_is_zero - test whether buffer is initialized
 * @buf: buffer to check
 * @from: check from this position
 * @to: check up until (excluding) this position
 *
 * This function returns true if the there is a non-zero byte
 * in the buf in the range [from,to).
 */
static inline bool range_is_zero(void *buf, size_t from, size_t to)
{
        return !memchr_inv((u8 *)buf + from, 0, to - from);
}

static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb)
{
        struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;

        if (!__skb)
                return 0;

        /* make sure the fields we don't use are zeroed */
        if (!range_is_zero(__skb, 0, offsetof(struct __sk_buff, mark)))
                return -EINVAL;

        /* mark is allowed */

        if (!range_is_zero(__skb, offsetofend(struct __sk_buff, mark),
                           offsetof(struct __sk_buff, priority)))
                return -EINVAL;

        /* priority is allowed */
        /* ingress_ifindex is allowed */
        /* ifindex is allowed */

        if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex),
                           offsetof(struct __sk_buff, cb)))
                return -EINVAL;

        /* cb is allowed */

        if (!range_is_zero(__skb, offsetofend(struct __sk_buff, cb),
                           offsetof(struct __sk_buff, data_end)))
                return -EINVAL;

        /* data_end is allowed, but not copied to skb */

        if (!range_is_zero(__skb, offsetofend(struct __sk_buff, data_end),
                           offsetof(struct __sk_buff, tstamp)))
                return -EINVAL;

        /* tstamp is allowed */
        /* wire_len is allowed */
        /* gso_segs is allowed */

        if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_segs),
                           offsetof(struct __sk_buff, gso_size)))
                return -EINVAL;

        /* gso_size is allowed */

        if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size),
                           offsetof(struct __sk_buff, hwtstamp)))
                return -EINVAL;

        /* hwtstamp is allowed */

        if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp),
                           sizeof(struct __sk_buff)))
                return -EINVAL;

        skb->mark = __skb->mark;
        skb->priority = __skb->priority;
        skb->skb_iif = __skb->ingress_ifindex;
        skb->tstamp = __skb->tstamp;
        memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);

        if (__skb->wire_len == 0) {
                cb->pkt_len = skb->len;
        } else {
                if (__skb->wire_len < skb->len ||
                    __skb->wire_len > GSO_LEGACY_MAX_SIZE)
                        return -EINVAL;
                cb->pkt_len = __skb->wire_len;
        }

        if (__skb->gso_segs > GSO_MAX_SEGS)
                return -EINVAL;

        /* Currently GSO type is zero/unset. If this gets extended with
         * a small list of accepted GSO types in future, the filter for
         * an unset GSO type in bpf_clone_redirect() can be lifted.
         */
        skb_shinfo(skb)->gso_segs = __skb->gso_segs;
        skb_shinfo(skb)->gso_size = __skb->gso_size;
        skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp;

        return 0;
}

static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb)
{
        struct qdisc_skb_cb *cb = (struct qdisc_skb_cb *)skb->cb;

        if (!__skb)
                return;

        __skb->mark = skb->mark;
        __skb->priority = skb->priority;
        __skb->ingress_ifindex = skb->skb_iif;
        __skb->ifindex = skb->dev->ifindex;
        __skb->tstamp = skb->tstamp;
        memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
        __skb->wire_len = cb->pkt_len;
        __skb->gso_segs = skb_shinfo(skb)->gso_segs;
        __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp;
}

static struct proto bpf_dummy_proto = {
        .name   = "bpf_dummy",
        .owner  = THIS_MODULE,
        .obj_size = sizeof(struct sock),
};

int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
                          union bpf_attr __user *uattr)
{
        bool is_l2 = false, is_direct_pkt_access = false, is_lwt = false;
        u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
        struct net *net = current->nsproxy->net_ns;
        struct net_device *dev = net->loopback_dev;
        u32 headroom = NET_SKB_PAD + NET_IP_ALIGN;
        u32 linear_sz = kattr->test.data_size_in;
        u32 repeat = kattr->test.repeat;
        struct __sk_buff *ctx = NULL;
        struct sk_buff *skb = NULL;
        struct sock *sk = NULL;
        u32 retval, duration;
        int hh_len = ETH_HLEN;
        void *data = NULL;
        int ret;

        if ((kattr->test.flags & ~BPF_F_TEST_SKB_CHECKSUM_COMPLETE) ||
            kattr->test.cpu || kattr->test.batch_size)
                return -EINVAL;

        if (kattr->test.data_size_in < ETH_HLEN)
                return -EINVAL;

        switch (prog->type) {
        case BPF_PROG_TYPE_SCHED_CLS:
        case BPF_PROG_TYPE_SCHED_ACT:
                is_direct_pkt_access = true;
                is_l2 = true;
                break;
        case BPF_PROG_TYPE_LWT_IN:
        case BPF_PROG_TYPE_LWT_OUT:
        case BPF_PROG_TYPE_LWT_XMIT:
                is_lwt = true;
                fallthrough;
        case BPF_PROG_TYPE_CGROUP_SKB:
                is_direct_pkt_access = true;
                break;
        default:
                break;
        }

        ctx = bpf_ctx_init(kattr, sizeof(struct __sk_buff));
        if (IS_ERR(ctx))
                return PTR_ERR(ctx);

        if (ctx) {
                if (ctx->data_end > kattr->test.data_size_in || ctx->data || ctx->data_meta) {
                        ret = -EINVAL;
                        goto out;
                }
                if (ctx->data_end) {
                        /* Non-linear LWT test_run is unsupported for now. */
                        if (is_lwt) {
                                ret = -EINVAL;
                                goto out;
                        }
                        linear_sz = max(ETH_HLEN, ctx->data_end);
                }
        }

        linear_sz = min_t(u32, linear_sz, PAGE_SIZE - headroom - tailroom);

        data = bpf_test_init(kattr, linear_sz, linear_sz, headroom, tailroom);
        if (IS_ERR(data)) {
                ret = PTR_ERR(data);
                data = NULL;
                goto out;
        }

        sk = sk_alloc(net, AF_UNSPEC, GFP_USER, &bpf_dummy_proto, 1);
        if (!sk) {
                ret = -ENOMEM;
                goto out;
        }
        sock_init_data(NULL, sk);

        skb = slab_build_skb(data);
        if (!skb) {
                ret = -ENOMEM;
                goto out;
        }
        skb->sk = sk;

        data = NULL; /* data released via kfree_skb */

        skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
        __skb_put(skb, linear_sz);

        if (unlikely(kattr->test.data_size_in > linear_sz)) {
                void __user *data_in = u64_to_user_ptr(kattr->test.data_in);
                struct skb_shared_info *sinfo = skb_shinfo(skb);
                u32 copied = linear_sz;

                while (copied < kattr->test.data_size_in) {
                        struct page *page;
                        u32 data_len;

                        if (sinfo->nr_frags == MAX_SKB_FRAGS) {
                                ret = -ENOMEM;
                                goto out;
                        }

                        page = alloc_page(GFP_KERNEL);
                        if (!page) {
                                ret = -ENOMEM;
                                goto out;
                        }

                        data_len = min_t(u32, kattr->test.data_size_in - copied,
                                         PAGE_SIZE);
                        skb_fill_page_desc(skb, sinfo->nr_frags, page, 0, data_len);

                        if (copy_from_user(page_address(page), data_in + copied,
                                           data_len)) {
                                ret = -EFAULT;
                                goto out;
                        }
                        skb->data_len += data_len;
                        skb->truesize += PAGE_SIZE;
                        skb->len += data_len;
                        copied += data_len;
                }
        }

        if (ctx && ctx->ifindex > 1) {
                dev = dev_get_by_index(net, ctx->ifindex);
                if (!dev) {
                        ret = -ENODEV;
                        goto out;
                }
        }
        skb->protocol = eth_type_trans(skb, dev);
        skb_reset_network_header(skb);

        switch (skb->protocol) {
        case htons(ETH_P_IP):
                if (skb_headlen(skb) < sizeof(struct iphdr)) {
                        ret = -EINVAL;
                        goto out;
                }
                sk->sk_family = AF_INET;
                sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
                sk->sk_daddr = ip_hdr(skb)->daddr;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                if (skb_headlen(skb) < sizeof(struct ipv6hdr)) {
                        ret = -EINVAL;
                        goto out;
                }
                sk->sk_family = AF_INET6;
                sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
                sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
                break;
#endif
        default:
                break;
        }

        if (is_l2)
                __skb_push(skb, hh_len);
        if (is_direct_pkt_access)
                bpf_compute_data_pointers(skb);

        ret = convert___skb_to_skb(skb, ctx);
        if (ret)
                goto out;

        if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) {
                const int off = skb_network_offset(skb);
                int len = skb->len - off;

                skb->csum = skb_checksum(skb, off, len, 0);
                skb->ip_summed = CHECKSUM_COMPLETE;
        }

        if (prog->type == BPF_PROG_TYPE_LWT_XMIT) {
                if (!ipv6_mod_enabled()) {
                        pr_warn_once("Please test this program with IPv6 enabled kernel\n");
                        ret = -EOPNOTSUPP;
                        goto out;
                }
#if IS_ENABLED(CONFIG_IPV6)
                dst_hold(&net->ipv6.ip6_null_entry->dst);
                skb_dst_set(skb, &net->ipv6.ip6_null_entry->dst);
#endif
        }

        ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false);
        if (ret)
                goto out;
        if (!is_l2) {
                if (skb_headroom(skb) < hh_len) {
                        int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb));

                        if (pskb_expand_head(skb, nhead, 0, GFP_USER)) {
                                ret = -ENOMEM;
                                goto out;
                        }
                }
                memset(__skb_push(skb, hh_len), 0, hh_len);
        }

        if (kattr->test.flags & BPF_F_TEST_SKB_CHECKSUM_COMPLETE) {
                const int off = skb_network_offset(skb);
                int len = skb->len - off;
                __wsum csum;

                csum = skb_checksum(skb, off, len, 0);

                if (csum_fold(skb->csum) != csum_fold(csum)) {
                        ret = -EBADMSG;
                        goto out;
                }
        }

        convert_skb_to___skb(skb, ctx);

        if (skb_is_nonlinear(skb))
                /* bpf program can never convert linear skb to non-linear */
                WARN_ON_ONCE(linear_sz == kattr->test.data_size_in);
        ret = bpf_test_finish(kattr, uattr, skb->data, skb_shinfo(skb), skb->len,
                              skb->data_len, retval, duration);
        if (!ret)
                ret = bpf_ctx_finish(kattr, uattr, ctx,
                                     sizeof(struct __sk_buff));
out:
        if (dev && dev != net->loopback_dev)
                dev_put(dev);
        kfree_skb(skb);
        kfree(data);
        if (sk)
                sk_free(sk);
        kfree(ctx);
        return ret;
}

static int xdp_convert_md_to_buff(struct xdp_md *xdp_md, struct xdp_buff *xdp)
{
        unsigned int ingress_ifindex, rx_queue_index;
        struct netdev_rx_queue *rxqueue;
        struct net_device *device;

        if (!xdp_md)
                return 0;

        if (xdp_md->egress_ifindex != 0)
                return -EINVAL;

        ingress_ifindex = xdp_md->ingress_ifindex;
        rx_queue_index = xdp_md->rx_queue_index;

        if (!ingress_ifindex && rx_queue_index)
                return -EINVAL;

        if (ingress_ifindex) {
                device = dev_get_by_index(current->nsproxy->net_ns,
                                          ingress_ifindex);
                if (!device)
                        return -ENODEV;

                if (rx_queue_index >= device->real_num_rx_queues)
                        goto free_dev;

                rxqueue = __netif_get_rx_queue(device, rx_queue_index);

                if (!xdp_rxq_info_is_reg(&rxqueue->xdp_rxq))
                        goto free_dev;

                xdp->rxq = &rxqueue->xdp_rxq;
                /* The device is now tracked in the xdp->rxq for later
                 * dev_put()
                 */
        }

        xdp->data = xdp->data_meta + xdp_md->data;
        return 0;

free_dev:
        dev_put(device);
        return -EINVAL;
}

static void xdp_convert_buff_to_md(struct xdp_buff *xdp, struct xdp_md *xdp_md)
{
        if (!xdp_md)
                return;

        xdp_md->data = xdp->data - xdp->data_meta;
        xdp_md->data_end = xdp->data_end - xdp->data_meta;

        if (xdp_md->ingress_ifindex)
                dev_put(xdp->rxq->dev);
}

int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
                          union bpf_attr __user *uattr)
{
        bool do_live = (kattr->test.flags & BPF_F_TEST_XDP_LIVE_FRAMES);
        u32 tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
        u32 retval = 0, meta_sz = 0, duration, max_linear_sz, size;
        u32 linear_sz = kattr->test.data_size_in;
        u32 batch_size = kattr->test.batch_size;
        u32 headroom = XDP_PACKET_HEADROOM;
        u32 repeat = kattr->test.repeat;
        struct netdev_rx_queue *rxqueue;
        struct skb_shared_info *sinfo;
        struct xdp_buff xdp = {};
        int i, ret = -EINVAL;
        struct xdp_md *ctx;
        void *data;

        if (prog->expected_attach_type == BPF_XDP_DEVMAP ||
            prog->expected_attach_type == BPF_XDP_CPUMAP)
                return -EINVAL;

        if (kattr->test.flags & ~BPF_F_TEST_XDP_LIVE_FRAMES)
                return -EINVAL;

        if (bpf_prog_is_dev_bound(prog->aux))
                return -EINVAL;

        if (do_live) {
                if (!batch_size)
                        batch_size = NAPI_POLL_WEIGHT;
                else if (batch_size > TEST_XDP_MAX_BATCH)
                        return -E2BIG;
        } else if (batch_size) {
                return -EINVAL;
        }

        ctx = bpf_ctx_init(kattr, sizeof(struct xdp_md));
        if (IS_ERR(ctx))
                return PTR_ERR(ctx);

        if (ctx) {
                /* There can't be user provided data before the meta data */
                if (ctx->data_meta || ctx->data_end > kattr->test.data_size_in ||
                    ctx->data > ctx->data_end ||
                    (do_live && (kattr->test.data_out || kattr->test.ctx_out)))
                        goto free_ctx;

                meta_sz = ctx->data;
                if (xdp_metalen_invalid(meta_sz) || meta_sz > headroom - sizeof(struct xdp_frame))
                        goto free_ctx;

                /* Meta data is allocated from the headroom */
                headroom -= meta_sz;
                linear_sz = ctx->data_end;
        }

        /* The xdp_page_head structure takes up space in each page, limiting the
         * size of the packet data; add the extra size to headroom here to make
         * sure it's accounted in the length checks below, but not in the
         * metadata size check above.
         */
        if (do_live)
                headroom += sizeof(struct xdp_page_head);

        max_linear_sz = PAGE_SIZE - headroom - tailroom;
        linear_sz = min_t(u32, linear_sz, max_linear_sz);

        /* disallow live data mode for jumbo frames */
        if (do_live && kattr->test.data_size_in > linear_sz)
                goto free_ctx;

        if (kattr->test.data_size_in - meta_sz < ETH_HLEN)
                goto free_ctx;

        data = bpf_test_init(kattr, linear_sz, max_linear_sz, headroom, tailroom);
        if (IS_ERR(data)) {
                ret = PTR_ERR(data);
                goto free_ctx;
        }

        rxqueue = __netif_get_rx_queue(current->nsproxy->net_ns->loopback_dev, 0);
        rxqueue->xdp_rxq.frag_size = PAGE_SIZE;
        xdp_init_buff(&xdp, rxqueue->xdp_rxq.frag_size, &rxqueue->xdp_rxq);
        xdp_prepare_buff(&xdp, data, headroom, linear_sz, true);
        sinfo = xdp_get_shared_info_from_buff(&xdp);

        ret = xdp_convert_md_to_buff(ctx, &xdp);
        if (ret)
                goto free_data;

        size = linear_sz;
        if (unlikely(kattr->test.data_size_in > size)) {
                void __user *data_in = u64_to_user_ptr(kattr->test.data_in);

                while (size < kattr->test.data_size_in) {
                        struct page *page;
                        skb_frag_t *frag;
                        u32 data_len;

                        if (sinfo->nr_frags == MAX_SKB_FRAGS) {
                                ret = -ENOMEM;
                                goto out_put_dev;
                        }

                        page = alloc_page(GFP_KERNEL);
                        if (!page) {
                                ret = -ENOMEM;
                                goto out_put_dev;
                        }

                        frag = &sinfo->frags[sinfo->nr_frags++];

                        data_len = min_t(u32, kattr->test.data_size_in - size,
                                         PAGE_SIZE);
                        skb_frag_fill_page_desc(frag, page, 0, data_len);

                        if (copy_from_user(page_address(page), data_in + size,
                                           data_len)) {
                                ret = -EFAULT;
                                goto out_put_dev;
                        }
                        sinfo->xdp_frags_size += data_len;
                        size += data_len;
                }
                xdp_buff_set_frags_flag(&xdp);
        }

        if (repeat > 1)
                bpf_prog_change_xdp(NULL, prog);

        if (do_live)
                ret = bpf_test_run_xdp_live(prog, &xdp, repeat, batch_size, &duration);
        else
                ret = bpf_test_run(prog, &xdp, repeat, &retval, &duration, true);
out_put_dev:
        /* We convert the xdp_buff back to an xdp_md before checking the return
         * code so the reference count of any held netdevice will be decremented
         * even if the test run failed.
         */
        xdp_convert_buff_to_md(&xdp, ctx);
        if (ret)
                goto out;

        size = xdp.data_end - xdp.data_meta + sinfo->xdp_frags_size;
        ret = bpf_test_finish(kattr, uattr, xdp.data_meta, sinfo, size, sinfo->xdp_frags_size,
                              retval, duration);
        if (!ret)
                ret = bpf_ctx_finish(kattr, uattr, ctx,
                                     sizeof(struct xdp_md));

out:
        if (repeat > 1)
                bpf_prog_change_xdp(prog, NULL);
free_data:
        for (i = 0; i < sinfo->nr_frags; i++)
                __free_page(skb_frag_page(&sinfo->frags[i]));
        kfree(data);
free_ctx:
        kfree(ctx);
        return ret;
}

static int verify_user_bpf_flow_keys(struct bpf_flow_keys *ctx)
{
        /* make sure the fields we don't use are zeroed */
        if (!range_is_zero(ctx, 0, offsetof(struct bpf_flow_keys, flags)))
                return -EINVAL;

        /* flags is allowed */

        if (!range_is_zero(ctx, offsetofend(struct bpf_flow_keys, flags),
                           sizeof(struct bpf_flow_keys)))
                return -EINVAL;

        return 0;
}

int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
                                     const union bpf_attr *kattr,
                                     union bpf_attr __user *uattr)
{
        struct bpf_test_timer t = {};
        u32 size = kattr->test.data_size_in;
        struct bpf_flow_dissector ctx = {};
        u32 repeat = kattr->test.repeat;
        struct bpf_flow_keys *user_ctx;
        struct bpf_flow_keys flow_keys;
        const struct ethhdr *eth;
        unsigned int flags = 0;
        u32 retval, duration;
        void *data;
        int ret;

        if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
                return -EINVAL;

        if (size < ETH_HLEN)
                return -EINVAL;

        data = bpf_test_init(kattr, kattr->test.data_size_in, size, 0, 0);
        if (IS_ERR(data))
                return PTR_ERR(data);

        eth = (struct ethhdr *)data;

        if (!repeat)
                repeat = 1;

        user_ctx = bpf_ctx_init(kattr, sizeof(struct bpf_flow_keys));
        if (IS_ERR(user_ctx)) {
                kfree(data);
                return PTR_ERR(user_ctx);
        }
        if (user_ctx) {
                ret = verify_user_bpf_flow_keys(user_ctx);
                if (ret)
                        goto out;
                flags = user_ctx->flags;
        }

        ctx.flow_keys = &flow_keys;
        ctx.data = data;
        ctx.data_end = (__u8 *)data + size;

        bpf_test_timer_enter(&t);
        do {
                retval = bpf_flow_dissect(prog, &ctx, eth->h_proto, ETH_HLEN,
                                          size, flags);
        } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration));
        bpf_test_timer_leave(&t);

        if (ret < 0)
                goto out;

        ret = bpf_test_finish(kattr, uattr, &flow_keys, NULL,
                              sizeof(flow_keys), 0, retval, duration);
        if (!ret)
                ret = bpf_ctx_finish(kattr, uattr, user_ctx,
                                     sizeof(struct bpf_flow_keys));

out:
        kfree(user_ctx);
        kfree(data);
        return ret;
}

int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr,
                                union bpf_attr __user *uattr)
{
        struct bpf_test_timer t = {};
        struct bpf_prog_array *progs = NULL;
        struct bpf_sk_lookup_kern ctx = {};
        u32 repeat = kattr->test.repeat;
        struct bpf_sk_lookup *user_ctx;
        u32 retval, duration;
        int ret = -EINVAL;

        if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
                return -EINVAL;

        if (kattr->test.data_in || kattr->test.data_size_in || kattr->test.data_out ||
            kattr->test.data_size_out)
                return -EINVAL;

        if (!repeat)
                repeat = 1;

        user_ctx = bpf_ctx_init(kattr, sizeof(*user_ctx));
        if (IS_ERR(user_ctx))
                return PTR_ERR(user_ctx);

        if (!user_ctx)
                return -EINVAL;

        if (user_ctx->sk)
                goto out;

        if (!range_is_zero(user_ctx, offsetofend(typeof(*user_ctx), local_port), sizeof(*user_ctx)))
                goto out;

        if (user_ctx->local_port > U16_MAX) {
                ret = -ERANGE;
                goto out;
        }

        ctx.family = (u16)user_ctx->family;
        ctx.protocol = (u16)user_ctx->protocol;
        ctx.dport = (u16)user_ctx->local_port;
        ctx.sport = user_ctx->remote_port;

        switch (ctx.family) {
        case AF_INET:
                ctx.v4.daddr = (__force __be32)user_ctx->local_ip4;
                ctx.v4.saddr = (__force __be32)user_ctx->remote_ip4;
                break;

#if IS_ENABLED(CONFIG_IPV6)
        case AF_INET6:
                ctx.v6.daddr = (struct in6_addr *)user_ctx->local_ip6;
                ctx.v6.saddr = (struct in6_addr *)user_ctx->remote_ip6;
                break;
#endif

        default:
                ret = -EAFNOSUPPORT;
                goto out;
        }

        progs = bpf_prog_array_alloc(1, GFP_KERNEL);
        if (!progs) {
                ret = -ENOMEM;
                goto out;
        }

        progs->items[0].prog = prog;

        bpf_test_timer_enter(&t);
        do {
                ctx.selected_sk = NULL;
                retval = BPF_PROG_SK_LOOKUP_RUN_ARRAY(progs, ctx, bpf_prog_run);
        } while (bpf_test_timer_continue(&t, 1, repeat, &ret, &duration));
        bpf_test_timer_leave(&t);

        if (ret < 0)
                goto out;

        user_ctx->cookie = 0;
        if (ctx.selected_sk) {
                if (ctx.selected_sk->sk_reuseport && !ctx.no_reuseport) {
                        ret = -EOPNOTSUPP;
                        goto out;
                }

                user_ctx->cookie = sock_gen_cookie(ctx.selected_sk);
        }

        ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, 0, retval, duration);
        if (!ret)
                ret = bpf_ctx_finish(kattr, uattr, user_ctx, sizeof(*user_ctx));

out:
        bpf_prog_array_free(progs);
        kfree(user_ctx);
        return ret;
}

int bpf_prog_test_run_syscall(struct bpf_prog *prog,
                              const union bpf_attr *kattr,
                              union bpf_attr __user *uattr)
{
        void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
        __u32 ctx_size_in = kattr->test.ctx_size_in;
        void *ctx = NULL;
        u32 retval;
        int err = 0;

        /* doesn't support data_in/out, ctx_out, duration, or repeat or flags */
        if (kattr->test.data_in || kattr->test.data_out ||
            kattr->test.ctx_out || kattr->test.duration ||
            kattr->test.repeat || kattr->test.flags ||
            kattr->test.batch_size)
                return -EINVAL;

        if (ctx_size_in < prog->aux->max_ctx_offset ||
            ctx_size_in > U16_MAX)
                return -EINVAL;

        if (ctx_size_in) {
                ctx = memdup_user(ctx_in, ctx_size_in);
                if (IS_ERR(ctx))
                        return PTR_ERR(ctx);
        }

        rcu_read_lock_trace();
        retval = bpf_prog_run_pin_on_cpu(prog, ctx);
        rcu_read_unlock_trace();

        if (copy_to_user(&uattr->test.retval, &retval, sizeof(u32))) {
                err = -EFAULT;
                goto out;
        }
        if (ctx_size_in)
                if (copy_to_user(ctx_in, ctx, ctx_size_in))
                        err = -EFAULT;
out:
        kfree(ctx);
        return err;
}

static int verify_and_copy_hook_state(struct nf_hook_state *state,
                                      const struct nf_hook_state *user,
                                      struct net_device *dev)
{
        if (user->in || user->out)
                return -EINVAL;

        if (user->net || user->sk || user->okfn)
                return -EINVAL;

        switch (user->pf) {
        case NFPROTO_IPV4:
        case NFPROTO_IPV6:
                switch (state->hook) {
                case NF_INET_PRE_ROUTING:
                        state->in = dev;
                        break;
                case NF_INET_LOCAL_IN:
                        state->in = dev;
                        break;
                case NF_INET_FORWARD:
                        state->in = dev;
                        state->out = dev;
                        break;
                case NF_INET_LOCAL_OUT:
                        state->out = dev;
                        break;
                case NF_INET_POST_ROUTING:
                        state->out = dev;
                        break;
                }

                break;
        default:
                return -EINVAL;
        }

        state->pf = user->pf;
        state->hook = user->hook;

        return 0;
}

static __be16 nfproto_eth(int nfproto)
{
        switch (nfproto) {
        case NFPROTO_IPV4:
                return htons(ETH_P_IP);
        case NFPROTO_IPV6:
                break;
        }

        return htons(ETH_P_IPV6);
}

int bpf_prog_test_run_nf(struct bpf_prog *prog,
                         const union bpf_attr *kattr,
                         union bpf_attr __user *uattr)
{
        struct net *net = current->nsproxy->net_ns;
        struct net_device *dev = net->loopback_dev;
        struct nf_hook_state *user_ctx, hook_state = {
                .pf = NFPROTO_IPV4,
                .hook = NF_INET_LOCAL_OUT,
        };
        u32 size = kattr->test.data_size_in;
        u32 repeat = kattr->test.repeat;
        struct bpf_nf_ctx ctx = {
                .state = &hook_state,
        };
        struct sk_buff *skb = NULL;
        u32 retval, duration;
        void *data;
        int ret;

        if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
                return -EINVAL;

        if (size < sizeof(struct iphdr))
                return -EINVAL;

        data = bpf_test_init(kattr, kattr->test.data_size_in, size,
                             NET_SKB_PAD + NET_IP_ALIGN,
                             SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
        if (IS_ERR(data))
                return PTR_ERR(data);

        if (!repeat)
                repeat = 1;

        user_ctx = bpf_ctx_init(kattr, sizeof(struct nf_hook_state));
        if (IS_ERR(user_ctx)) {
                kfree(data);
                return PTR_ERR(user_ctx);
        }

        if (user_ctx) {
                ret = verify_and_copy_hook_state(&hook_state, user_ctx, dev);
                if (ret)
                        goto out;
        }

        skb = slab_build_skb(data);
        if (!skb) {
                ret = -ENOMEM;
                goto out;
        }

        data = NULL; /* data released via kfree_skb */

        skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
        __skb_put(skb, size);

        ret = -EINVAL;

        if (hook_state.hook != NF_INET_LOCAL_OUT) {
                if (size < ETH_HLEN + sizeof(struct iphdr))
                        goto out;

                skb->protocol = eth_type_trans(skb, dev);
                switch (skb->protocol) {
                case htons(ETH_P_IP):
                        if (hook_state.pf == NFPROTO_IPV4)
                                break;
                        goto out;
                case htons(ETH_P_IPV6):
                        if (size < ETH_HLEN + sizeof(struct ipv6hdr))
                                goto out;
                        if (hook_state.pf == NFPROTO_IPV6)
                                break;
                        goto out;
                default:
                        ret = -EPROTO;
                        goto out;
                }

                skb_reset_network_header(skb);
        } else {
                skb->protocol = nfproto_eth(hook_state.pf);
        }

        ctx.skb = skb;

        ret = bpf_test_run(prog, &ctx, repeat, &retval, &duration, false);
        if (ret)
                goto out;

        ret = bpf_test_finish(kattr, uattr, NULL, NULL, 0, 0, retval, duration);

out:
        kfree(user_ctx);
        kfree_skb(skb);
        kfree(data);
        return ret;
}

static const struct btf_kfunc_id_set bpf_prog_test_kfunc_set = {
        .owner = THIS_MODULE,
        .set   = &test_sk_check_kfunc_ids,
};

BTF_ID_LIST(bpf_prog_test_dtor_kfunc_ids)
BTF_ID(struct, prog_test_ref_kfunc)
BTF_ID(func, bpf_kfunc_call_test_release_dtor)
BTF_ID(struct, prog_test_member)
BTF_ID(func, bpf_kfunc_call_memb_release_dtor)

static int __init bpf_prog_test_run_init(void)
{
        const struct btf_id_dtor_kfunc bpf_prog_test_dtor_kfunc[] = {
                {
                  .btf_id       = bpf_prog_test_dtor_kfunc_ids[0],
                  .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[1]
                },
                {
                  .btf_id        = bpf_prog_test_dtor_kfunc_ids[2],
                  .kfunc_btf_id = bpf_prog_test_dtor_kfunc_ids[3],
                },
        };
        int ret;

        ret = register_btf_fmodret_id_set(&bpf_test_modify_return_set);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_prog_test_kfunc_set);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_prog_test_kfunc_set);
        return ret ?: register_btf_id_dtor_kfuncs(bpf_prog_test_dtor_kfunc,
                                                  ARRAY_SIZE(bpf_prog_test_dtor_kfunc),
                                                  THIS_MODULE);
}
late_initcall(bpf_prog_test_run_init);






















































































   22 




   22 
   22 



   22 














































































































































































































































































    7 
    7 














    7 







    7 





































































































































































































































































































































































































































































































































































































    2 









   10 























    4 




    4 










































































    3 

















   23 























   23 




   23 






   12 



   23 





   10 


   10 









  798 




    4 










    5 


  798 








    5 




















  788 















  788 























    9 




























   10 








    9 
   10 


    4 












    5 


    4 




    1 

















    1 




    4 






    9 









    9 














    9 
    9 
    9 








    9 


    9 









































  613 

























  613 











  241 
  305 


























  549 












  549 


















  479 



  479 










































































   76 




  310 



























 1357 









  549 
  549 







 1354 
  722 


  741 



  335 





  183 

   76 




  221 


   53 










  240 





   64 

   64 

   64 











































   15 










   15 










   23 










    3 







    4 




    8 




    8 






    2 
    2 




    3 
    3 





    3 

































   35 





    2 

    2 










    2 



    2 



    2 



    2 










    2 




    2 




    2 












   17 

   17 























    4 

    2 











    1 


















    2 












   15 









   12 



    3 










   20 









   16 



    4 




































































































































































































   21 











   22 

   21 






















   22 


















   22 
















































































    1 





   10 











































   11 


















   11 
   10 




    2 


    5 





    1 





    2 

































































































































































































































































































   30 
























   30 

































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/types.h>
#include <linux/skbuff.h>
#include <linux/socket.h>
#include <linux/sysctl.h>
#include <linux/net.h>
#include <linux/module.h>
#include <linux/if_arp.h>
#include <linux/ipv6.h>
#include <linux/mpls.h>
#include <linux/netconf.h>
#include <linux/nospec.h>
#include <linux/vmalloc.h>
#include <linux/percpu.h>
#include <net/gso.h>
#include <net/ip.h>
#include <net/dst.h>
#include <net/sock.h>
#include <net/arp.h>
#include <net/ip_fib.h>
#include <net/netevent.h>
#include <net/ip_tunnels.h>
#include <net/netns/generic.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ipv6.h>
#endif
#include <net/rtnh.h>
#include "internal.h"

/* max memory we will use for mpls_route */
#define MAX_MPLS_ROUTE_MEM        4096

/* Maximum number of labels to look ahead at when selecting a path of
 * a multipath route
 */
#define MAX_MP_SELECT_LABELS 4

#define MPLS_NEIGH_TABLE_UNSPEC (NEIGH_LINK_TABLE + 1)

static int label_limit = (1 << 20) - 1;
static int ttl_max = 255;

#if IS_ENABLED(CONFIG_NET_IP_TUNNEL)
static size_t ipgre_mpls_encap_hlen(struct ip_tunnel_encap *e)
{
        return sizeof(struct mpls_shim_hdr);
}

static const struct ip_tunnel_encap_ops mpls_iptun_ops = {
        .encap_hlen        = ipgre_mpls_encap_hlen,
};

static int ipgre_tunnel_encap_add_mpls_ops(void)
{
        return ip_tunnel_encap_add_ops(&mpls_iptun_ops, TUNNEL_ENCAP_MPLS);
}

static void ipgre_tunnel_encap_del_mpls_ops(void)
{
        ip_tunnel_encap_del_ops(&mpls_iptun_ops, TUNNEL_ENCAP_MPLS);
}
#else
static int ipgre_tunnel_encap_add_mpls_ops(void)
{
        return 0;
}

static void ipgre_tunnel_encap_del_mpls_ops(void)
{
}
#endif

static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
                       struct nlmsghdr *nlh, struct net *net, u32 portid,
                       unsigned int nlm_flags);

static struct mpls_route *mpls_route_input(struct net *net, unsigned int index)
{
        struct mpls_route __rcu **platform_label;

        platform_label = mpls_dereference(net, net->mpls.platform_label);
        return mpls_dereference(net, platform_label[index]);
}

static struct mpls_route __rcu **mpls_platform_label_rcu(struct net *net, size_t *platform_labels)
{
        struct mpls_route __rcu **platform_label;
        unsigned int sequence;

        do {
                sequence = read_seqcount_begin(&net->mpls.platform_label_seq);
                platform_label = rcu_dereference(net->mpls.platform_label);
                *platform_labels = net->mpls.platform_labels;
        } while (read_seqcount_retry(&net->mpls.platform_label_seq, sequence));

        return platform_label;
}

static struct mpls_route *mpls_route_input_rcu(struct net *net, unsigned int index)
{
        struct mpls_route __rcu **platform_label;
        size_t platform_labels;

        platform_label = mpls_platform_label_rcu(net, &platform_labels);

        if (index >= platform_labels)
                return NULL;

        return rcu_dereference(platform_label[index]);
}

bool mpls_output_possible(const struct net_device *dev)
{
        return dev && (dev->flags & IFF_UP) && netif_carrier_ok(dev);
}
EXPORT_SYMBOL_GPL(mpls_output_possible);

static u8 *__mpls_nh_via(struct mpls_route *rt, struct mpls_nh *nh)
{
        return (u8 *)nh + rt->rt_via_offset;
}

static const u8 *mpls_nh_via(const struct mpls_route *rt,
                             const struct mpls_nh *nh)
{
        return __mpls_nh_via((struct mpls_route *)rt, (struct mpls_nh *)nh);
}

static unsigned int mpls_nh_header_size(const struct mpls_nh *nh)
{
        /* The size of the layer 2.5 labels to be added for this route */
        return nh->nh_labels * sizeof(struct mpls_shim_hdr);
}

unsigned int mpls_dev_mtu(const struct net_device *dev)
{
        /* The amount of data the layer 2 frame can hold */
        return dev->mtu;
}
EXPORT_SYMBOL_GPL(mpls_dev_mtu);

bool mpls_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
{
        if (skb->len <= mtu)
                return false;

        if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
                return false;

        return true;
}
EXPORT_SYMBOL_GPL(mpls_pkt_too_big);

void mpls_stats_inc_outucastpkts(struct net *net,
                                 struct net_device *dev,
                                 const struct sk_buff *skb)
{
        struct mpls_dev *mdev;

        if (skb->protocol == htons(ETH_P_MPLS_UC)) {
                mdev = mpls_dev_rcu(dev);
                if (mdev)
                        MPLS_INC_STATS_LEN(mdev, skb->len,
                                           tx_packets,
                                           tx_bytes);
        } else if (skb->protocol == htons(ETH_P_IP)) {
                IP_UPD_PO_STATS(net, IPSTATS_MIB_OUT, skb->len);
#if IS_ENABLED(CONFIG_IPV6)
        } else if (skb->protocol == htons(ETH_P_IPV6)) {
                struct inet6_dev *in6dev = in6_dev_rcu(dev);

                if (in6dev)
                        IP6_UPD_PO_STATS(net, in6dev,
                                         IPSTATS_MIB_OUT, skb->len);
#endif
        }
}
EXPORT_SYMBOL_GPL(mpls_stats_inc_outucastpkts);

static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb)
{
        struct mpls_entry_decoded dec;
        unsigned int mpls_hdr_len = 0;
        struct mpls_shim_hdr *hdr;
        bool eli_seen = false;
        int label_index;
        u32 hash = 0;

        for (label_index = 0; label_index < MAX_MP_SELECT_LABELS;
             label_index++) {
                mpls_hdr_len += sizeof(*hdr);
                if (!pskb_may_pull(skb, mpls_hdr_len))
                        break;

                /* Read and decode the current label */
                hdr = mpls_hdr(skb) + label_index;
                dec = mpls_entry_decode(hdr);

                /* RFC6790 - reserved labels MUST NOT be used as keys
                 * for the load-balancing function
                 */
                if (likely(dec.label >= MPLS_LABEL_FIRST_UNRESERVED)) {
                        hash = jhash_1word(dec.label, hash);

                        /* The entropy label follows the entropy label
                         * indicator, so this means that the entropy
                         * label was just added to the hash - no need to
                         * go any deeper either in the label stack or in the
                         * payload
                         */
                        if (eli_seen)
                                break;
                } else if (dec.label == MPLS_LABEL_ENTROPY) {
                        eli_seen = true;
                }

                if (!dec.bos)
                        continue;

                /* found bottom label; does skb have room for a header? */
                if (pskb_may_pull(skb, mpls_hdr_len + sizeof(struct iphdr))) {
                        const struct iphdr *v4hdr;

                        v4hdr = (const struct iphdr *)(hdr + 1);
                        if (v4hdr->version == 4) {
                                hash = jhash_3words(ntohl(v4hdr->saddr),
                                                    ntohl(v4hdr->daddr),
                                                    v4hdr->protocol, hash);
                        } else if (v4hdr->version == 6 &&
                                   pskb_may_pull(skb, mpls_hdr_len +
                                                 sizeof(struct ipv6hdr))) {
                                const struct ipv6hdr *v6hdr;

                                v6hdr = (const struct ipv6hdr *)(hdr + 1);
                                hash = __ipv6_addr_jhash(&v6hdr->saddr, hash);
                                hash = __ipv6_addr_jhash(&v6hdr->daddr, hash);
                                hash = jhash_1word(v6hdr->nexthdr, hash);
                        }
                }

                break;
        }

        return hash;
}

static struct mpls_nh *mpls_get_nexthop(struct mpls_route *rt, u8 index)
{
        return (struct mpls_nh *)((u8 *)rt->rt_nh + index * rt->rt_nh_size);
}

/* number of alive nexthops (rt->rt_nhn_alive) and the flags for
 * a next hop (nh->nh_flags) are modified by netdev event handlers.
 * Since those fields can change at any moment, use READ_ONCE to
 * access both.
 */
static const struct mpls_nh *mpls_select_multipath(struct mpls_route *rt,
                                                   struct sk_buff *skb)
{
        u32 hash = 0;
        int nh_index = 0;
        int n = 0;
        u8 alive;

        /* No need to look further into packet if there's only
         * one path
         */
        if (rt->rt_nhn == 1)
                return rt->rt_nh;

        alive = READ_ONCE(rt->rt_nhn_alive);
        if (alive == 0)
                return NULL;

        hash = mpls_multipath_hash(rt, skb);
        nh_index = hash % alive;
        if (alive == rt->rt_nhn)
                goto out;
        for_nexthops(rt) {
                unsigned int nh_flags = READ_ONCE(nh->nh_flags);

                if (nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
                        continue;
                if (n == nh_index)
                        return nh;
                n++;
        } endfor_nexthops(rt);

out:
        return mpls_get_nexthop(rt, nh_index);
}

static bool mpls_egress(struct net *net, struct mpls_route *rt,
                        struct sk_buff *skb, struct mpls_entry_decoded dec)
{
        enum mpls_payload_type payload_type;
        bool success = false;

        /* The IPv4 code below accesses through the IPv4 header
         * checksum, which is 12 bytes into the packet.
         * The IPv6 code below accesses through the IPv6 hop limit
         * which is 8 bytes into the packet.
         *
         * For all supported cases there should always be at least 12
         * bytes of packet data present.  The IPv4 header is 20 bytes
         * without options and the IPv6 header is always 40 bytes
         * long.
         */
        if (!pskb_may_pull(skb, 12))
                return false;

        payload_type = rt->rt_payload_type;
        if (payload_type == MPT_UNSPEC)
                payload_type = ip_hdr(skb)->version;

        switch (payload_type) {
        case MPT_IPV4: {
                struct iphdr *hdr4 = ip_hdr(skb);
                u8 new_ttl;
                skb->protocol = htons(ETH_P_IP);

                /* If propagating TTL, take the decremented TTL from
                 * the incoming MPLS header, otherwise decrement the
                 * TTL, but only if not 0 to avoid underflow.
                 */
                if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
                    (rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
                     net->mpls.ip_ttl_propagate))
                        new_ttl = dec.ttl;
                else
                        new_ttl = hdr4->ttl ? hdr4->ttl - 1 : 0;

                csum_replace2(&hdr4->check,
                              htons(hdr4->ttl << 8),
                              htons(new_ttl << 8));
                hdr4->ttl = new_ttl;
                success = true;
                break;
        }
        case MPT_IPV6: {
                struct ipv6hdr *hdr6 = ipv6_hdr(skb);
                skb->protocol = htons(ETH_P_IPV6);

                /* If propagating TTL, take the decremented TTL from
                 * the incoming MPLS header, otherwise decrement the
                 * hop limit, but only if not 0 to avoid underflow.
                 */
                if (rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED ||
                    (rt->rt_ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
                     net->mpls.ip_ttl_propagate))
                        hdr6->hop_limit = dec.ttl;
                else if (hdr6->hop_limit)
                        hdr6->hop_limit = hdr6->hop_limit - 1;
                success = true;
                break;
        }
        case MPT_UNSPEC:
                /* Should have decided which protocol it is by now */
                break;
        }

        return success;
}

static int mpls_forward(struct sk_buff *skb, struct net_device *dev,
                        struct packet_type *pt, struct net_device *orig_dev)
{
        struct net *net = dev_net_rcu(dev);
        struct mpls_shim_hdr *hdr;
        const struct mpls_nh *nh;
        struct mpls_route *rt;
        struct mpls_entry_decoded dec;
        struct net_device *out_dev;
        struct mpls_dev *out_mdev;
        struct mpls_dev *mdev;
        unsigned int hh_len;
        unsigned int new_header_size;
        unsigned int mtu;
        int err;

        /* Careful this entire function runs inside of an rcu critical section */

        mdev = mpls_dev_rcu(dev);
        if (!mdev)
                goto drop;

        MPLS_INC_STATS_LEN(mdev, skb->len, rx_packets,
                           rx_bytes);

        if (!mdev->input_enabled) {
                MPLS_INC_STATS(mdev, rx_dropped);
                goto drop;
        }

        if (skb->pkt_type != PACKET_HOST)
                goto err;

        if ((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)
                goto err;

        if (!pskb_may_pull(skb, sizeof(*hdr)))
                goto err;

        skb_dst_drop(skb);

        /* Read and decode the label */
        hdr = mpls_hdr(skb);
        dec = mpls_entry_decode(hdr);

        rt = mpls_route_input_rcu(net, dec.label);
        if (!rt) {
                MPLS_INC_STATS(mdev, rx_noroute);
                goto drop;
        }

        nh = mpls_select_multipath(rt, skb);
        if (!nh)
                goto err;

        /* Pop the label */
        skb_pull(skb, sizeof(*hdr));
        skb_reset_network_header(skb);

        skb_orphan(skb);

        if (skb_warn_if_lro(skb))
                goto err;

        skb_forward_csum(skb);

        /* Verify ttl is valid */
        if (dec.ttl <= 1)
                goto err;

        /* Find the output device */
        out_dev = nh->nh_dev;
        if (!mpls_output_possible(out_dev))
                goto tx_err;

        /* Verify the destination can hold the packet */
        new_header_size = mpls_nh_header_size(nh);
        mtu = mpls_dev_mtu(out_dev);
        if (mpls_pkt_too_big(skb, mtu - new_header_size))
                goto tx_err;

        hh_len = LL_RESERVED_SPACE(out_dev);
        if (!out_dev->header_ops)
                hh_len = 0;

        /* Ensure there is enough space for the headers in the skb */
        if (skb_cow(skb, hh_len + new_header_size))
                goto tx_err;

        skb->dev = out_dev;
        skb->protocol = htons(ETH_P_MPLS_UC);

        dec.ttl -= 1;
        if (unlikely(!new_header_size && dec.bos)) {
                /* Penultimate hop popping */
                if (!mpls_egress(net, rt, skb, dec))
                        goto err;
        } else {
                bool bos;
                int i;
                skb_push(skb, new_header_size);
                skb_reset_network_header(skb);
                /* Push the new labels */
                hdr = mpls_hdr(skb);
                bos = dec.bos;
                for (i = nh->nh_labels - 1; i >= 0; i--) {
                        hdr[i] = mpls_entry_encode(nh->nh_label[i],
                                                   dec.ttl, 0, bos);
                        bos = false;
                }
        }

        mpls_stats_inc_outucastpkts(net, out_dev, skb);

        /* If via wasn't specified then send out using device address */
        if (nh->nh_via_table == MPLS_NEIGH_TABLE_UNSPEC)
                err = neigh_xmit(NEIGH_LINK_TABLE, out_dev,
                                 out_dev->dev_addr, skb);
        else
                err = neigh_xmit(nh->nh_via_table, out_dev,
                                 mpls_nh_via(rt, nh), skb);
        if (err)
                net_dbg_ratelimited("%s: packet transmission failed: %d\n",
                                    __func__, err);
        return 0;

tx_err:
        out_mdev = out_dev ? mpls_dev_rcu(out_dev) : NULL;
        if (out_mdev)
                MPLS_INC_STATS(out_mdev, tx_errors);
        goto drop;
err:
        MPLS_INC_STATS(mdev, rx_errors);
drop:
        kfree_skb(skb);
        return NET_RX_DROP;
}

static struct packet_type mpls_packet_type __read_mostly = {
        .type = cpu_to_be16(ETH_P_MPLS_UC),
        .func = mpls_forward,
};

static const struct nla_policy rtm_mpls_policy[RTA_MAX+1] = {
        [RTA_DST]                = { .type = NLA_U32 },
        [RTA_OIF]                = { .type = NLA_U32 },
        [RTA_TTL_PROPAGATE]        = { .type = NLA_U8 },
};

struct mpls_route_config {
        u32                        rc_protocol;
        u32                        rc_ifindex;
        u8                        rc_via_table;
        u8                        rc_via_alen;
        u8                        rc_via[MAX_VIA_ALEN];
        u32                        rc_label;
        u8                        rc_ttl_propagate;
        u8                        rc_output_labels;
        u32                        rc_output_label[MAX_NEW_LABELS];
        u32                        rc_nlflags;
        enum mpls_payload_type        rc_payload_type;
        struct nl_info                rc_nlinfo;
        struct rtnexthop        *rc_mp;
        int                        rc_mp_len;
};

/* all nexthops within a route have the same size based on max
 * number of labels and max via length for a hop
 */
static struct mpls_route *mpls_rt_alloc(u8 num_nh, u8 max_alen, u8 max_labels)
{
        u8 nh_size = MPLS_NH_SIZE(max_labels, max_alen);
        struct mpls_route *rt;
        size_t size;

        size = sizeof(*rt) + num_nh * nh_size;
        if (size > MAX_MPLS_ROUTE_MEM)
                return ERR_PTR(-EINVAL);

        rt = kzalloc(size, GFP_KERNEL);
        if (!rt)
                return ERR_PTR(-ENOMEM);

        rt->rt_nhn = num_nh;
        rt->rt_nhn_alive = num_nh;
        rt->rt_nh_size = nh_size;
        rt->rt_via_offset = MPLS_NH_VIA_OFF(max_labels);

        return rt;
}

static void mpls_rt_free_rcu(struct rcu_head *head)
{
        struct mpls_route *rt;

        rt = container_of(head, struct mpls_route, rt_rcu);

        change_nexthops(rt) {
                netdev_put(nh->nh_dev, &nh->nh_dev_tracker);
        } endfor_nexthops(rt);

        kfree(rt);
}

static void mpls_rt_free(struct mpls_route *rt)
{
        if (rt)
                call_rcu(&rt->rt_rcu, mpls_rt_free_rcu);
}

static void mpls_notify_route(struct net *net, unsigned index,
                              struct mpls_route *old, struct mpls_route *new,
                              const struct nl_info *info)
{
        struct nlmsghdr *nlh = info ? info->nlh : NULL;
        unsigned portid = info ? info->portid : 0;
        int event = new ? RTM_NEWROUTE : RTM_DELROUTE;
        struct mpls_route *rt = new ? new : old;
        unsigned nlm_flags = (old && new) ? NLM_F_REPLACE : 0;
        /* Ignore reserved labels for now */
        if (rt && (index >= MPLS_LABEL_FIRST_UNRESERVED))
                rtmsg_lfib(event, index, rt, nlh, net, portid, nlm_flags);
}

static void mpls_route_update(struct net *net, unsigned index,
                              struct mpls_route *new,
                              const struct nl_info *info)
{
        struct mpls_route __rcu **platform_label;
        struct mpls_route *rt;

        platform_label = mpls_dereference(net, net->mpls.platform_label);
        rt = mpls_dereference(net, platform_label[index]);
        rcu_assign_pointer(platform_label[index], new);

        mpls_notify_route(net, index, rt, new, info);

        /* If we removed a route free it now */
        mpls_rt_free(rt);
}

static unsigned int find_free_label(struct net *net)
{
        unsigned int index;

        for (index = MPLS_LABEL_FIRST_UNRESERVED;
             index < net->mpls.platform_labels;
             index++) {
                if (!mpls_route_input(net, index))
                        return index;
        }

        return LABEL_NOT_SPECIFIED;
}

#if IS_ENABLED(CONFIG_INET)
static struct net_device *inet_fib_lookup_dev(struct net *net,
                                              struct mpls_nh *nh,
                                              const void *addr)
{
        struct net_device *dev;
        struct rtable *rt;
        struct in_addr daddr;

        memcpy(&daddr, addr, sizeof(struct in_addr));
        rt = ip_route_output(net, daddr.s_addr, 0, 0, 0, RT_SCOPE_UNIVERSE);
        if (IS_ERR(rt))
                return ERR_CAST(rt);

        dev = rt->dst.dev;
        netdev_hold(dev, &nh->nh_dev_tracker, GFP_KERNEL);
        ip_rt_put(rt);

        return dev;
}
#else
static struct net_device *inet_fib_lookup_dev(struct net *net,
                                              struct mpls_nh *nh,
                                              const void *addr)
{
        return ERR_PTR(-EAFNOSUPPORT);
}
#endif

#if IS_ENABLED(CONFIG_IPV6)
static struct net_device *inet6_fib_lookup_dev(struct net *net,
                                               struct mpls_nh *nh,
                                               const void *addr)
{
        struct net_device *dev;
        struct dst_entry *dst;
        struct flowi6 fl6;

        memset(&fl6, 0, sizeof(fl6));
        memcpy(&fl6.daddr, addr, sizeof(struct in6_addr));
        dst = ip6_dst_lookup_flow(net, NULL, &fl6, NULL);
        if (IS_ERR(dst))
                return ERR_CAST(dst);

        dev = dst->dev;
        netdev_hold(dev, &nh->nh_dev_tracker, GFP_KERNEL);
        dst_release(dst);

        return dev;
}
#else
static struct net_device *inet6_fib_lookup_dev(struct net *net,
                                               struct mpls_nh *nh,
                                               const void *addr)
{
        return ERR_PTR(-EAFNOSUPPORT);
}
#endif

static struct net_device *find_outdev(struct net *net,
                                      struct mpls_route *rt,
                                      struct mpls_nh *nh, int oif)
{
        struct net_device *dev = NULL;

        if (!oif) {
                switch (nh->nh_via_table) {
                case NEIGH_ARP_TABLE:
                        dev = inet_fib_lookup_dev(net, nh, mpls_nh_via(rt, nh));
                        break;
                case NEIGH_ND_TABLE:
                        dev = inet6_fib_lookup_dev(net, nh, mpls_nh_via(rt, nh));
                        break;
                case NEIGH_LINK_TABLE:
                        break;
                }
        } else {
                dev = netdev_get_by_index(net, oif,
                                          &nh->nh_dev_tracker, GFP_KERNEL);
        }

        if (!dev)
                return ERR_PTR(-ENODEV);

        if (IS_ERR(dev))
                return dev;

        nh->nh_dev = dev;

        return dev;
}

static int mpls_nh_assign_dev(struct net *net, struct mpls_route *rt,
                              struct mpls_nh *nh, int oif)
{
        struct net_device *dev = NULL;
        int err = -ENODEV;

        dev = find_outdev(net, rt, nh, oif);
        if (IS_ERR(dev)) {
                err = PTR_ERR(dev);
                goto errout;
        }

        /* Ensure this is a supported device */
        err = -EINVAL;
        if (!mpls_dev_get(net, dev))
                goto errout_put;

        if ((nh->nh_via_table == NEIGH_LINK_TABLE) &&
            (dev->addr_len != nh->nh_via_alen))
                goto errout_put;

        if (!(dev->flags & IFF_UP)) {
                nh->nh_flags |= RTNH_F_DEAD;
        } else {
                unsigned int flags;

                flags = netif_get_flags(dev);
                if (!(flags & (IFF_RUNNING | IFF_LOWER_UP)))
                        nh->nh_flags |= RTNH_F_LINKDOWN;
        }

        return 0;

errout_put:
        netdev_put(nh->nh_dev, &nh->nh_dev_tracker);
        nh->nh_dev = NULL;
errout:
        return err;
}

static int nla_get_via(const struct nlattr *nla, u8 *via_alen, u8 *via_table,
                       u8 via_addr[], struct netlink_ext_ack *extack)
{
        struct rtvia *via = nla_data(nla);
        int err = -EINVAL;
        int alen;

        if (nla_len(nla) < offsetof(struct rtvia, rtvia_addr)) {
                NL_SET_ERR_MSG_ATTR(extack, nla,
                                    "Invalid attribute length for RTA_VIA");
                goto errout;
        }
        alen = nla_len(nla) -
                        offsetof(struct rtvia, rtvia_addr);
        if (alen > MAX_VIA_ALEN) {
                NL_SET_ERR_MSG_ATTR(extack, nla,
                                    "Invalid address length for RTA_VIA");
                goto errout;
        }

        /* Validate the address family */
        switch (via->rtvia_family) {
        case AF_PACKET:
                *via_table = NEIGH_LINK_TABLE;
                break;
        case AF_INET:
                *via_table = NEIGH_ARP_TABLE;
                if (alen != 4)
                        goto errout;
                break;
        case AF_INET6:
                *via_table = NEIGH_ND_TABLE;
                if (alen != 16)
                        goto errout;
                break;
        default:
                /* Unsupported address family */
                goto errout;
        }

        memcpy(via_addr, via->rtvia_addr, alen);
        *via_alen = alen;
        err = 0;

errout:
        return err;
}

static int mpls_nh_build_from_cfg(struct mpls_route_config *cfg,
                                  struct mpls_route *rt)
{
        struct net *net = cfg->rc_nlinfo.nl_net;
        struct mpls_nh *nh = rt->rt_nh;
        int err;
        int i;

        if (!nh)
                return -ENOMEM;

        nh->nh_labels = cfg->rc_output_labels;
        for (i = 0; i < nh->nh_labels; i++)
                nh->nh_label[i] = cfg->rc_output_label[i];

        nh->nh_via_table = cfg->rc_via_table;
        memcpy(__mpls_nh_via(rt, nh), cfg->rc_via, cfg->rc_via_alen);
        nh->nh_via_alen = cfg->rc_via_alen;

        err = mpls_nh_assign_dev(net, rt, nh, cfg->rc_ifindex);
        if (err)
                goto errout;

        if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
                rt->rt_nhn_alive--;

        return 0;

errout:
        return err;
}

static int mpls_nh_build(struct net *net, struct mpls_route *rt,
                         struct mpls_nh *nh, int oif, struct nlattr *via,
                         struct nlattr *newdst, u8 max_labels,
                         struct netlink_ext_ack *extack)
{
        int err = -ENOMEM;

        if (!nh)
                goto errout;

        if (newdst) {
                err = nla_get_labels(newdst, max_labels, &nh->nh_labels,
                                     nh->nh_label, extack);
                if (err)
                        goto errout;
        }

        if (via) {
                err = nla_get_via(via, &nh->nh_via_alen, &nh->nh_via_table,
                                  __mpls_nh_via(rt, nh), extack);
                if (err)
                        goto errout;
        } else {
                nh->nh_via_table = MPLS_NEIGH_TABLE_UNSPEC;
        }

        err = mpls_nh_assign_dev(net, rt, nh, oif);
        if (err)
                goto errout;

        return 0;

errout:
        return err;
}

static u8 mpls_count_nexthops(struct rtnexthop *rtnh, int len,
                              u8 cfg_via_alen, u8 *max_via_alen,
                              u8 *max_labels)
{
        int remaining = len;
        u8 nhs = 0;

        *max_via_alen = 0;
        *max_labels = 0;

        while (rtnh_ok(rtnh, remaining)) {
                struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
                int attrlen;
                u8 n_labels = 0;

                attrlen = rtnh_attrlen(rtnh);
                nla = nla_find(attrs, attrlen, RTA_VIA);
                if (nla && nla_len(nla) >=
                    offsetof(struct rtvia, rtvia_addr)) {
                        int via_alen = nla_len(nla) -
                                offsetof(struct rtvia, rtvia_addr);

                        if (via_alen <= MAX_VIA_ALEN)
                                *max_via_alen = max_t(u16, *max_via_alen,
                                                      via_alen);
                }

                nla = nla_find(attrs, attrlen, RTA_NEWDST);
                if (nla &&
                    nla_get_labels(nla, MAX_NEW_LABELS, &n_labels,
                                   NULL, NULL) != 0)
                        return 0;

                *max_labels = max_t(u8, *max_labels, n_labels);

                /* number of nexthops is tracked by a u8.
                 * Check for overflow.
                 */
                if (nhs == 255)
                        return 0;
                nhs++;

                rtnh = rtnh_next(rtnh, &remaining);
        }

        /* leftover implies invalid nexthop configuration, discard it */
        return remaining > 0 ? 0 : nhs;
}

static int mpls_nh_build_multi(struct mpls_route_config *cfg,
                               struct mpls_route *rt, u8 max_labels,
                               struct netlink_ext_ack *extack)
{
        struct rtnexthop *rtnh = cfg->rc_mp;
        struct nlattr *nla_via, *nla_newdst;
        int remaining = cfg->rc_mp_len;
        int err = 0;

        rt->rt_nhn = 0;

        change_nexthops(rt) {
                int attrlen;

                nla_via = NULL;
                nla_newdst = NULL;

                err = -EINVAL;
                if (!rtnh_ok(rtnh, remaining))
                        goto errout;

                /* neither weighted multipath nor any flags
                 * are supported
                 */
                if (rtnh->rtnh_hops || rtnh->rtnh_flags)
                        goto errout;

                attrlen = rtnh_attrlen(rtnh);
                if (attrlen > 0) {
                        struct nlattr *attrs = rtnh_attrs(rtnh);

                        nla_via = nla_find(attrs, attrlen, RTA_VIA);
                        nla_newdst = nla_find(attrs, attrlen, RTA_NEWDST);
                }

                err = mpls_nh_build(cfg->rc_nlinfo.nl_net, rt, nh,
                                    rtnh->rtnh_ifindex, nla_via, nla_newdst,
                                    max_labels, extack);
                if (err)
                        goto errout;

                if (nh->nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN))
                        rt->rt_nhn_alive--;

                rtnh = rtnh_next(rtnh, &remaining);
                rt->rt_nhn++;
        } endfor_nexthops(rt);

        return 0;

errout:
        return err;
}

static bool mpls_label_ok(struct net *net, unsigned int *index,
                          struct netlink_ext_ack *extack)
{
        /* Reserved labels may not be set */
        if (*index < MPLS_LABEL_FIRST_UNRESERVED) {
                NL_SET_ERR_MSG(extack,
                               "Invalid label - must be MPLS_LABEL_FIRST_UNRESERVED or higher");
                return false;
        }

        /* The full 20 bit range may not be supported. */
        if (*index >= net->mpls.platform_labels) {
                NL_SET_ERR_MSG(extack,
                               "Label >= configured maximum in platform_labels");
                return false;
        }

        *index = array_index_nospec(*index, net->mpls.platform_labels);

        return true;
}

static int mpls_route_add(struct mpls_route_config *cfg,
                          struct netlink_ext_ack *extack)
{
        struct net *net = cfg->rc_nlinfo.nl_net;
        struct mpls_route *rt, *old;
        int err = -EINVAL;
        u8 max_via_alen;
        unsigned index;
        u8 max_labels;
        u8 nhs;

        index = cfg->rc_label;

        /* If a label was not specified during insert pick one */
        if ((index == LABEL_NOT_SPECIFIED) &&
            (cfg->rc_nlflags & NLM_F_CREATE)) {
                index = find_free_label(net);
        }

        if (!mpls_label_ok(net, &index, extack))
                goto errout;

        /* Append makes no sense with mpls */
        err = -EOPNOTSUPP;
        if (cfg->rc_nlflags & NLM_F_APPEND) {
                NL_SET_ERR_MSG(extack, "MPLS does not support route append");
                goto errout;
        }

        err = -EEXIST;
        old = mpls_route_input(net, index);
        if ((cfg->rc_nlflags & NLM_F_EXCL) && old)
                goto errout;

        err = -EEXIST;
        if (!(cfg->rc_nlflags & NLM_F_REPLACE) && old)
                goto errout;

        err = -ENOENT;
        if (!(cfg->rc_nlflags & NLM_F_CREATE) && !old)
                goto errout;

        err = -EINVAL;
        if (cfg->rc_mp) {
                nhs = mpls_count_nexthops(cfg->rc_mp, cfg->rc_mp_len,
                                          cfg->rc_via_alen, &max_via_alen,
                                          &max_labels);
        } else {
                max_via_alen = cfg->rc_via_alen;
                max_labels = cfg->rc_output_labels;
                nhs = 1;
        }

        if (nhs == 0) {
                NL_SET_ERR_MSG(extack, "Route does not contain a nexthop");
                goto errout;
        }

        rt = mpls_rt_alloc(nhs, max_via_alen, max_labels);
        if (IS_ERR(rt)) {
                err = PTR_ERR(rt);
                goto errout;
        }

        rt->rt_protocol = cfg->rc_protocol;
        rt->rt_payload_type = cfg->rc_payload_type;
        rt->rt_ttl_propagate = cfg->rc_ttl_propagate;

        if (cfg->rc_mp)
                err = mpls_nh_build_multi(cfg, rt, max_labels, extack);
        else
                err = mpls_nh_build_from_cfg(cfg, rt);
        if (err)
                goto freert;

        mpls_route_update(net, index, rt, &cfg->rc_nlinfo);

        return 0;

freert:
        mpls_rt_free(rt);
errout:
        return err;
}

static int mpls_route_del(struct mpls_route_config *cfg,
                          struct netlink_ext_ack *extack)
{
        struct net *net = cfg->rc_nlinfo.nl_net;
        unsigned index;
        int err = -EINVAL;

        index = cfg->rc_label;

        if (!mpls_label_ok(net, &index, extack))
                goto errout;

        mpls_route_update(net, index, NULL, &cfg->rc_nlinfo);

        err = 0;
errout:
        return err;
}

static void mpls_get_stats(struct mpls_dev *mdev,
                           struct mpls_link_stats *stats)
{
        struct mpls_pcpu_stats *p;
        int i;

        memset(stats, 0, sizeof(*stats));

        for_each_possible_cpu(i) {
                struct mpls_link_stats local;
                unsigned int start;

                p = per_cpu_ptr(mdev->stats, i);
                do {
                        start = u64_stats_fetch_begin(&p->syncp);
                        local = p->stats;
                } while (u64_stats_fetch_retry(&p->syncp, start));

                stats->rx_packets        += local.rx_packets;
                stats->rx_bytes                += local.rx_bytes;
                stats->tx_packets        += local.tx_packets;
                stats->tx_bytes                += local.tx_bytes;
                stats->rx_errors        += local.rx_errors;
                stats->tx_errors        += local.tx_errors;
                stats->rx_dropped        += local.rx_dropped;
                stats->tx_dropped        += local.tx_dropped;
                stats->rx_noroute        += local.rx_noroute;
        }
}

static int mpls_fill_stats_af(struct sk_buff *skb,
                              const struct net_device *dev)
{
        struct mpls_link_stats *stats;
        struct mpls_dev *mdev;
        struct nlattr *nla;

        mdev = mpls_dev_rcu(dev);
        if (!mdev)
                return -ENODATA;

        nla = nla_reserve_64bit(skb, MPLS_STATS_LINK,
                                sizeof(struct mpls_link_stats),
                                MPLS_STATS_UNSPEC);
        if (!nla)
                return -EMSGSIZE;

        stats = nla_data(nla);
        mpls_get_stats(mdev, stats);

        return 0;
}

static size_t mpls_get_stats_af_size(const struct net_device *dev)
{
        struct mpls_dev *mdev;

        mdev = mpls_dev_rcu(dev);
        if (!mdev)
                return 0;

        return nla_total_size_64bit(sizeof(struct mpls_link_stats));
}

static int mpls_netconf_fill_devconf(struct sk_buff *skb, struct mpls_dev *mdev,
                                     u32 portid, u32 seq, int event,
                                     unsigned int flags, int type)
{
        struct nlmsghdr  *nlh;
        struct netconfmsg *ncm;
        bool all = false;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct netconfmsg),
                        flags);
        if (!nlh)
                return -EMSGSIZE;

        if (type == NETCONFA_ALL)
                all = true;

        ncm = nlmsg_data(nlh);
        ncm->ncm_family = AF_MPLS;

        if (nla_put_s32(skb, NETCONFA_IFINDEX, mdev->dev->ifindex) < 0)
                goto nla_put_failure;

        if ((all || type == NETCONFA_INPUT) &&
            nla_put_s32(skb, NETCONFA_INPUT,
                        READ_ONCE(mdev->input_enabled)) < 0)
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int mpls_netconf_msgsize_devconf(int type)
{
        int size = NLMSG_ALIGN(sizeof(struct netconfmsg))
                        + nla_total_size(4); /* NETCONFA_IFINDEX */
        bool all = false;

        if (type == NETCONFA_ALL)
                all = true;

        if (all || type == NETCONFA_INPUT)
                size += nla_total_size(4);

        return size;
}

static void mpls_netconf_notify_devconf(struct net *net, int event,
                                        int type, struct mpls_dev *mdev)
{
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(mpls_netconf_msgsize_devconf(type), GFP_KERNEL);
        if (!skb)
                goto errout;

        err = mpls_netconf_fill_devconf(skb, mdev, 0, 0, event, 0, type);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, net, 0, RTNLGRP_MPLS_NETCONF, NULL, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_MPLS_NETCONF, err);
}

static const struct nla_policy devconf_mpls_policy[NETCONFA_MAX + 1] = {
        [NETCONFA_IFINDEX]        = { .len = sizeof(int) },
};

static int mpls_netconf_valid_get_req(struct sk_buff *skb,
                                      const struct nlmsghdr *nlh,
                                      struct nlattr **tb,
                                      struct netlink_ext_ack *extack)
{
        int i, err;

        if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(struct netconfmsg))) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid header for netconf get request");
                return -EINVAL;
        }

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(struct netconfmsg),
                                              tb, NETCONFA_MAX,
                                              devconf_mpls_policy, extack);

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct netconfmsg),
                                            tb, NETCONFA_MAX,
                                            devconf_mpls_policy, extack);
        if (err)
                return err;

        for (i = 0; i <= NETCONFA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case NETCONFA_IFINDEX:
                        break;
                default:
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in netconf get request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int mpls_netconf_get_devconf(struct sk_buff *in_skb,
                                    struct nlmsghdr *nlh,
                                    struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        struct nlattr *tb[NETCONFA_MAX + 1];
        struct net_device *dev;
        struct mpls_dev *mdev;
        struct sk_buff *skb;
        int ifindex;
        int err;

        err = mpls_netconf_valid_get_req(in_skb, nlh, tb, extack);
        if (err < 0)
                goto errout;

        if (!tb[NETCONFA_IFINDEX]) {
                err = -EINVAL;
                goto errout;
        }

        ifindex = nla_get_s32(tb[NETCONFA_IFINDEX]);

        skb = nlmsg_new(mpls_netconf_msgsize_devconf(NETCONFA_ALL), GFP_KERNEL);
        if (!skb) {
                err = -ENOBUFS;
                goto errout;
        }

        rcu_read_lock();

        dev = dev_get_by_index_rcu(net, ifindex);
        if (!dev) {
                err = -EINVAL;
                goto errout_unlock;
        }

        mdev = mpls_dev_rcu(dev);
        if (!mdev) {
                err = -EINVAL;
                goto errout_unlock;
        }

        err = mpls_netconf_fill_devconf(skb, mdev,
                                        NETLINK_CB(in_skb).portid,
                                        nlh->nlmsg_seq, RTM_NEWNETCONF, 0,
                                        NETCONFA_ALL);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in mpls_netconf_msgsize_devconf() */
                WARN_ON(err == -EMSGSIZE);
                goto errout_unlock;
        }

        err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);

        rcu_read_unlock();
errout:
        return err;

errout_unlock:
        rcu_read_unlock();
        kfree_skb(skb);
        goto errout;
}

static int mpls_netconf_dump_devconf(struct sk_buff *skb,
                                     struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        struct {
                unsigned long ifindex;
        } *ctx = (void *)cb->ctx;
        struct net_device *dev;
        struct mpls_dev *mdev;
        int err = 0;

        if (cb->strict_check) {
                struct netlink_ext_ack *extack = cb->extack;
                struct netconfmsg *ncm;

                if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ncm))) {
                        NL_SET_ERR_MSG_MOD(extack, "Invalid header for netconf dump request");
                        return -EINVAL;
                }

                if (nlmsg_attrlen(nlh, sizeof(*ncm))) {
                        NL_SET_ERR_MSG_MOD(extack, "Invalid data after header in netconf dump request");
                        return -EINVAL;
                }
        }

        rcu_read_lock();
        for_each_netdev_dump(net, dev, ctx->ifindex) {
                mdev = mpls_dev_rcu(dev);
                if (!mdev)
                        continue;
                err = mpls_netconf_fill_devconf(skb, mdev,
                                                NETLINK_CB(cb->skb).portid,
                                                nlh->nlmsg_seq,
                                                RTM_NEWNETCONF,
                                                NLM_F_MULTI,
                                                NETCONFA_ALL);
                if (err < 0)
                        break;
        }
        rcu_read_unlock();

        return err;
}

#define MPLS_PERDEV_SYSCTL_OFFSET(field)        \
        (&((struct mpls_dev *)0)->field)

static int mpls_conf_proc(const struct ctl_table *ctl, int write,
                          void *buffer, size_t *lenp, loff_t *ppos)
{
        int oval = *(int *)ctl->data;
        int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);

        if (write) {
                struct mpls_dev *mdev = ctl->extra1;
                int i = (int *)ctl->data - (int *)mdev;
                struct net *net = ctl->extra2;
                int val = *(int *)ctl->data;

                if (i == offsetof(struct mpls_dev, input_enabled) &&
                    val != oval) {
                        mpls_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_INPUT, mdev);
                }
        }

        return ret;
}

static const struct ctl_table mpls_dev_table[] = {
        {
                .procname        = "input",
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = mpls_conf_proc,
                .data                = MPLS_PERDEV_SYSCTL_OFFSET(input_enabled),
        },
};

static int mpls_dev_sysctl_register(struct net_device *dev,
                                    struct mpls_dev *mdev)
{
        char path[sizeof("net/mpls/conf/") + IFNAMSIZ];
        size_t table_size = ARRAY_SIZE(mpls_dev_table);
        struct net *net = dev_net(dev);
        struct ctl_table *table;
        int i;

        table = kmemdup(&mpls_dev_table, sizeof(mpls_dev_table), GFP_KERNEL);
        if (!table)
                goto out;

        /* Table data contains only offsets relative to the base of
         * the mdev at this point, so make them absolute.
         */
        for (i = 0; i < table_size; i++) {
                table[i].data = (char *)mdev + (uintptr_t)table[i].data;
                table[i].extra1 = mdev;
                table[i].extra2 = net;
        }

        snprintf(path, sizeof(path), "net/mpls/conf/%s", dev->name);

        mdev->sysctl = register_net_sysctl_sz(net, path, table, table_size);
        if (!mdev->sysctl)
                goto free;

        mpls_netconf_notify_devconf(net, RTM_NEWNETCONF, NETCONFA_ALL, mdev);
        return 0;

free:
        kfree(table);
out:
        mdev->sysctl = NULL;
        return -ENOBUFS;
}

static void mpls_dev_sysctl_unregister(struct net_device *dev,
                                       struct mpls_dev *mdev)
{
        struct net *net = dev_net(dev);
        const struct ctl_table *table;

        if (!mdev->sysctl)
                return;

        table = mdev->sysctl->ctl_table_arg;
        unregister_net_sysctl_table(mdev->sysctl);
        kfree(table);

        mpls_netconf_notify_devconf(net, RTM_DELNETCONF, 0, mdev);
}

static struct mpls_dev *mpls_add_dev(struct net_device *dev)
{
        struct mpls_dev *mdev;
        int err = -ENOMEM;
        int i;

        mdev = kzalloc_obj(*mdev);
        if (!mdev)
                return ERR_PTR(err);

        mdev->stats = alloc_percpu(struct mpls_pcpu_stats);
        if (!mdev->stats)
                goto free;

        for_each_possible_cpu(i) {
                struct mpls_pcpu_stats *mpls_stats;

                mpls_stats = per_cpu_ptr(mdev->stats, i);
                u64_stats_init(&mpls_stats->syncp);
        }

        mdev->dev = dev;

        err = mpls_dev_sysctl_register(dev, mdev);
        if (err)
                goto free;

        rcu_assign_pointer(dev->mpls_ptr, mdev);

        return mdev;

free:
        free_percpu(mdev->stats);
        kfree(mdev);
        return ERR_PTR(err);
}

static void mpls_dev_destroy_rcu(struct rcu_head *head)
{
        struct mpls_dev *mdev = container_of(head, struct mpls_dev, rcu);

        free_percpu(mdev->stats);
        kfree(mdev);
}

static int mpls_ifdown(struct net_device *dev, int event)
{
        struct net *net = dev_net(dev);
        unsigned int index;

        for (index = 0; index < net->mpls.platform_labels; index++) {
                struct mpls_route *rt;
                bool nh_del = false;
                u8 alive = 0;

                rt = mpls_route_input(net, index);
                if (!rt)
                        continue;

                if (event == NETDEV_UNREGISTER) {
                        u8 deleted = 0;

                        for_nexthops(rt) {
                                if (!nh->nh_dev || nh->nh_dev == dev)
                                        deleted++;
                                if (nh->nh_dev == dev)
                                        nh_del = true;
                        } endfor_nexthops(rt);

                        /* if there are no more nexthops, delete the route */
                        if (deleted == rt->rt_nhn) {
                                mpls_route_update(net, index, NULL, NULL);
                                continue;
                        }

                        if (nh_del) {
                                size_t size = sizeof(*rt) + rt->rt_nhn *
                                        rt->rt_nh_size;
                                struct mpls_route *orig = rt;

                                rt = kmemdup(orig, size, GFP_KERNEL);
                                if (!rt)
                                        return -ENOMEM;
                        }
                }

                change_nexthops(rt) {
                        unsigned int nh_flags = nh->nh_flags;

                        if (nh->nh_dev != dev) {
                                if (nh_del)
                                        netdev_hold(nh->nh_dev, &nh->nh_dev_tracker,
                                                    GFP_KERNEL);
                                goto next;
                        }

                        switch (event) {
                        case NETDEV_DOWN:
                        case NETDEV_UNREGISTER:
                                nh_flags |= RTNH_F_DEAD;
                                fallthrough;
                        case NETDEV_CHANGE:
                                nh_flags |= RTNH_F_LINKDOWN;
                                break;
                        }
                        if (event == NETDEV_UNREGISTER)
                                nh->nh_dev = NULL;

                        if (nh->nh_flags != nh_flags)
                                WRITE_ONCE(nh->nh_flags, nh_flags);
next:
                        if (!(nh_flags & (RTNH_F_DEAD | RTNH_F_LINKDOWN)))
                                alive++;
                } endfor_nexthops(rt);

                WRITE_ONCE(rt->rt_nhn_alive, alive);

                if (nh_del)
                        mpls_route_update(net, index, rt, NULL);
        }

        return 0;
}

static void mpls_ifup(struct net_device *dev, unsigned int flags)
{
        struct net *net = dev_net(dev);
        unsigned int index;
        u8 alive;

        for (index = 0; index < net->mpls.platform_labels; index++) {
                struct mpls_route *rt;

                rt = mpls_route_input(net, index);
                if (!rt)
                        continue;

                alive = 0;
                change_nexthops(rt) {
                        unsigned int nh_flags = nh->nh_flags;

                        if (!(nh_flags & flags)) {
                                alive++;
                                continue;
                        }
                        if (nh->nh_dev != dev)
                                continue;
                        alive++;
                        nh_flags &= ~flags;
                        WRITE_ONCE(nh->nh_flags, nh_flags);
                } endfor_nexthops(rt);

                WRITE_ONCE(rt->rt_nhn_alive, alive);
        }
}

static int mpls_dev_notify(struct notifier_block *this, unsigned long event,
                           void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct net *net = dev_net(dev);
        struct mpls_dev *mdev;
        unsigned int flags;
        int err;

        mutex_lock(&net->mpls.platform_mutex);

        if (event == NETDEV_REGISTER) {
                mdev = mpls_add_dev(dev);
                if (IS_ERR(mdev)) {
                        err = PTR_ERR(mdev);
                        goto err;
                }

                goto out;
        }

        mdev = mpls_dev_get(net, dev);
        if (!mdev)
                goto out;

        switch (event) {

        case NETDEV_DOWN:
                err = mpls_ifdown(dev, event);
                if (err)
                        goto err;
                break;
        case NETDEV_UP:
                flags = netif_get_flags(dev);
                if (flags & (IFF_RUNNING | IFF_LOWER_UP))
                        mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
                else
                        mpls_ifup(dev, RTNH_F_DEAD);
                break;
        case NETDEV_CHANGE:
                flags = netif_get_flags(dev);
                if (flags & (IFF_RUNNING | IFF_LOWER_UP)) {
                        mpls_ifup(dev, RTNH_F_DEAD | RTNH_F_LINKDOWN);
                } else {
                        err = mpls_ifdown(dev, event);
                        if (err)
                                goto err;
                }
                break;
        case NETDEV_UNREGISTER:
                err = mpls_ifdown(dev, event);
                if (err)
                        goto err;

                mdev = mpls_dev_get(net, dev);
                if (mdev) {
                        mpls_dev_sysctl_unregister(dev, mdev);
                        RCU_INIT_POINTER(dev->mpls_ptr, NULL);
                        call_rcu(&mdev->rcu, mpls_dev_destroy_rcu);
                }
                break;
        case NETDEV_CHANGENAME:
                mdev = mpls_dev_get(net, dev);
                if (mdev) {
                        mpls_dev_sysctl_unregister(dev, mdev);
                        err = mpls_dev_sysctl_register(dev, mdev);
                        if (err)
                                goto err;
                }
                break;
        }

out:
        mutex_unlock(&net->mpls.platform_mutex);
        return NOTIFY_OK;

err:
        mutex_unlock(&net->mpls.platform_mutex);
        return notifier_from_errno(err);
}

static struct notifier_block mpls_dev_notifier = {
        .notifier_call = mpls_dev_notify,
};

static int nla_put_via(struct sk_buff *skb,
                       u8 table, const void *addr, int alen)
{
        static const int table_to_family[NEIGH_NR_TABLES + 1] = {
                AF_INET, AF_INET6, AF_PACKET,
        };
        struct nlattr *nla;
        struct rtvia *via;
        int family = AF_UNSPEC;

        nla = nla_reserve(skb, RTA_VIA, alen + 2);
        if (!nla)
                return -EMSGSIZE;

        if (table <= NEIGH_NR_TABLES)
                family = table_to_family[table];

        via = nla_data(nla);
        via->rtvia_family = family;
        memcpy(via->rtvia_addr, addr, alen);
        return 0;
}

int nla_put_labels(struct sk_buff *skb, int attrtype,
                   u8 labels, const u32 label[])
{
        struct nlattr *nla;
        struct mpls_shim_hdr *nla_label;
        bool bos;
        int i;
        nla = nla_reserve(skb, attrtype, labels*4);
        if (!nla)
                return -EMSGSIZE;

        nla_label = nla_data(nla);
        bos = true;
        for (i = labels - 1; i >= 0; i--) {
                nla_label[i] = mpls_entry_encode(label[i], 0, 0, bos);
                bos = false;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(nla_put_labels);

int nla_get_labels(const struct nlattr *nla, u8 max_labels, u8 *labels,
                   u32 label[], struct netlink_ext_ack *extack)
{
        unsigned len = nla_len(nla);
        struct mpls_shim_hdr *nla_label;
        u8 nla_labels;
        bool bos;
        int i;

        /* len needs to be an even multiple of 4 (the label size). Number
         * of labels is a u8 so check for overflow.
         */
        if (len & 3 || len / 4 > 255) {
                NL_SET_ERR_MSG_ATTR(extack, nla,
                                    "Invalid length for labels attribute");
                return -EINVAL;
        }

        /* Limit the number of new labels allowed */
        nla_labels = len/4;
        if (nla_labels > max_labels) {
                NL_SET_ERR_MSG(extack, "Too many labels");
                return -EINVAL;
        }

        /* when label == NULL, caller wants number of labels */
        if (!label)
                goto out;

        nla_label = nla_data(nla);
        bos = true;
        for (i = nla_labels - 1; i >= 0; i--, bos = false) {
                struct mpls_entry_decoded dec;
                dec = mpls_entry_decode(nla_label + i);

                /* Ensure the bottom of stack flag is properly set
                 * and ttl and tc are both clear.
                 */
                if (dec.ttl) {
                        NL_SET_ERR_MSG_ATTR(extack, nla,
                                            "TTL in label must be 0");
                        return -EINVAL;
                }

                if (dec.tc) {
                        NL_SET_ERR_MSG_ATTR(extack, nla,
                                            "Traffic class in label must be 0");
                        return -EINVAL;
                }

                if (dec.bos != bos) {
                        NL_SET_BAD_ATTR(extack, nla);
                        if (bos) {
                                NL_SET_ERR_MSG(extack,
                                               "BOS bit must be set in first label");
                        } else {
                                NL_SET_ERR_MSG(extack,
                                               "BOS bit can only be set in first label");
                        }
                        return -EINVAL;
                }

                switch (dec.label) {
                case MPLS_LABEL_IMPLNULL:
                        /* RFC3032: This is a label that an LSR may
                         * assign and distribute, but which never
                         * actually appears in the encapsulation.
                         */
                        NL_SET_ERR_MSG_ATTR(extack, nla,
                                            "Implicit NULL Label (3) can not be used in encapsulation");
                        return -EINVAL;
                }

                label[i] = dec.label;
        }
out:
        *labels = nla_labels;
        return 0;
}
EXPORT_SYMBOL_GPL(nla_get_labels);

static int rtm_to_route_config(struct sk_buff *skb,
                               struct nlmsghdr *nlh,
                               struct mpls_route_config *cfg,
                               struct netlink_ext_ack *extack)
{
        struct rtmsg *rtm;
        struct nlattr *tb[RTA_MAX+1];
        int index;
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
                                     rtm_mpls_policy, extack);
        if (err < 0)
                goto errout;

        err = -EINVAL;
        rtm = nlmsg_data(nlh);

        if (rtm->rtm_family != AF_MPLS) {
                NL_SET_ERR_MSG(extack, "Invalid address family in rtmsg");
                goto errout;
        }
        if (rtm->rtm_dst_len != 20) {
                NL_SET_ERR_MSG(extack, "rtm_dst_len must be 20 for MPLS");
                goto errout;
        }
        if (rtm->rtm_src_len != 0) {
                NL_SET_ERR_MSG(extack, "rtm_src_len must be 0 for MPLS");
                goto errout;
        }
        if (rtm->rtm_tos != 0) {
                NL_SET_ERR_MSG(extack, "rtm_tos must be 0 for MPLS");
                goto errout;
        }
        if (rtm->rtm_table != RT_TABLE_MAIN) {
                NL_SET_ERR_MSG(extack,
                               "MPLS only supports the main route table");
                goto errout;
        }
        /* Any value is acceptable for rtm_protocol */

        /* As mpls uses destination specific addresses
         * (or source specific address in the case of multicast)
         * all addresses have universal scope.
         */
        if (rtm->rtm_scope != RT_SCOPE_UNIVERSE) {
                NL_SET_ERR_MSG(extack,
                               "Invalid route scope  - MPLS only supports UNIVERSE");
                goto errout;
        }
        if (rtm->rtm_type != RTN_UNICAST) {
                NL_SET_ERR_MSG(extack,
                               "Invalid route type - MPLS only supports UNICAST");
                goto errout;
        }
        if (rtm->rtm_flags != 0) {
                NL_SET_ERR_MSG(extack, "rtm_flags must be 0 for MPLS");
                goto errout;
        }

        cfg->rc_label                = LABEL_NOT_SPECIFIED;
        cfg->rc_protocol        = rtm->rtm_protocol;
        cfg->rc_via_table        = MPLS_NEIGH_TABLE_UNSPEC;
        cfg->rc_ttl_propagate        = MPLS_TTL_PROP_DEFAULT;
        cfg->rc_nlflags                = nlh->nlmsg_flags;
        cfg->rc_nlinfo.portid        = NETLINK_CB(skb).portid;
        cfg->rc_nlinfo.nlh        = nlh;
        cfg->rc_nlinfo.nl_net        = sock_net(skb->sk);

        for (index = 0; index <= RTA_MAX; index++) {
                struct nlattr *nla = tb[index];
                if (!nla)
                        continue;

                switch (index) {
                case RTA_OIF:
                        cfg->rc_ifindex = nla_get_u32(nla);
                        break;
                case RTA_NEWDST:
                        if (nla_get_labels(nla, MAX_NEW_LABELS,
                                           &cfg->rc_output_labels,
                                           cfg->rc_output_label, extack))
                                goto errout;
                        break;
                case RTA_DST:
                {
                        u8 label_count;
                        if (nla_get_labels(nla, 1, &label_count,
                                           &cfg->rc_label, extack))
                                goto errout;

                        if (!mpls_label_ok(cfg->rc_nlinfo.nl_net,
                                           &cfg->rc_label, extack))
                                goto errout;
                        break;
                }
                case RTA_GATEWAY:
                        NL_SET_ERR_MSG(extack, "MPLS does not support RTA_GATEWAY attribute");
                        goto errout;
                case RTA_VIA:
                {
                        if (nla_get_via(nla, &cfg->rc_via_alen,
                                        &cfg->rc_via_table, cfg->rc_via,
                                        extack))
                                goto errout;
                        break;
                }
                case RTA_MULTIPATH:
                {
                        cfg->rc_mp = nla_data(nla);
                        cfg->rc_mp_len = nla_len(nla);
                        break;
                }
                case RTA_TTL_PROPAGATE:
                {
                        u8 ttl_propagate = nla_get_u8(nla);

                        if (ttl_propagate > 1) {
                                NL_SET_ERR_MSG_ATTR(extack, nla,
                                                    "RTA_TTL_PROPAGATE can only be 0 or 1");
                                goto errout;
                        }
                        cfg->rc_ttl_propagate = ttl_propagate ?
                                MPLS_TTL_PROP_ENABLED :
                                MPLS_TTL_PROP_DISABLED;
                        break;
                }
                default:
                        NL_SET_ERR_MSG_ATTR(extack, nla, "Unknown attribute");
                        /* Unsupported attribute */
                        goto errout;
                }
        }

        err = 0;
errout:
        return err;
}

static int mpls_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
                             struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct mpls_route_config *cfg;
        int err;

        cfg = kzalloc_obj(*cfg);
        if (!cfg)
                return -ENOMEM;

        err = rtm_to_route_config(skb, nlh, cfg, extack);
        if (err < 0)
                goto out;

        mutex_lock(&net->mpls.platform_mutex);
        err = mpls_route_del(cfg, extack);
        mutex_unlock(&net->mpls.platform_mutex);
out:
        kfree(cfg);

        return err;
}


static int mpls_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
                             struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct mpls_route_config *cfg;
        int err;

        cfg = kzalloc_obj(*cfg);
        if (!cfg)
                return -ENOMEM;

        err = rtm_to_route_config(skb, nlh, cfg, extack);
        if (err < 0)
                goto out;

        mutex_lock(&net->mpls.platform_mutex);
        err = mpls_route_add(cfg, extack);
        mutex_unlock(&net->mpls.platform_mutex);
out:
        kfree(cfg);

        return err;
}

static int mpls_dump_route(struct sk_buff *skb, u32 portid, u32 seq, int event,
                           u32 label, struct mpls_route *rt, int flags)
{
        struct net_device *dev;
        struct nlmsghdr *nlh;
        struct rtmsg *rtm;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*rtm), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        rtm = nlmsg_data(nlh);
        rtm->rtm_family = AF_MPLS;
        rtm->rtm_dst_len = 20;
        rtm->rtm_src_len = 0;
        rtm->rtm_tos = 0;
        rtm->rtm_table = RT_TABLE_MAIN;
        rtm->rtm_protocol = rt->rt_protocol;
        rtm->rtm_scope = RT_SCOPE_UNIVERSE;
        rtm->rtm_type = RTN_UNICAST;
        rtm->rtm_flags = 0;

        if (nla_put_labels(skb, RTA_DST, 1, &label))
                goto nla_put_failure;

        if (rt->rt_ttl_propagate != MPLS_TTL_PROP_DEFAULT) {
                bool ttl_propagate =
                        rt->rt_ttl_propagate == MPLS_TTL_PROP_ENABLED;

                if (nla_put_u8(skb, RTA_TTL_PROPAGATE,
                               ttl_propagate))
                        goto nla_put_failure;
        }
        if (rt->rt_nhn == 1) {
                const struct mpls_nh *nh = rt->rt_nh;

                if (nh->nh_labels &&
                    nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
                                   nh->nh_label))
                        goto nla_put_failure;
                if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
                    nla_put_via(skb, nh->nh_via_table, mpls_nh_via(rt, nh),
                                nh->nh_via_alen))
                        goto nla_put_failure;
                dev = nh->nh_dev;
                if (dev && nla_put_u32(skb, RTA_OIF, dev->ifindex))
                        goto nla_put_failure;
                if (nh->nh_flags & RTNH_F_LINKDOWN)
                        rtm->rtm_flags |= RTNH_F_LINKDOWN;
                if (nh->nh_flags & RTNH_F_DEAD)
                        rtm->rtm_flags |= RTNH_F_DEAD;
        } else {
                struct rtnexthop *rtnh;
                struct nlattr *mp;
                u8 linkdown = 0;
                u8 dead = 0;

                mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
                if (!mp)
                        goto nla_put_failure;

                for_nexthops(rt) {
                        dev = nh->nh_dev;
                        if (!dev)
                                continue;

                        rtnh = nla_reserve_nohdr(skb, sizeof(*rtnh));
                        if (!rtnh)
                                goto nla_put_failure;

                        rtnh->rtnh_ifindex = dev->ifindex;
                        if (nh->nh_flags & RTNH_F_LINKDOWN) {
                                rtnh->rtnh_flags |= RTNH_F_LINKDOWN;
                                linkdown++;
                        }
                        if (nh->nh_flags & RTNH_F_DEAD) {
                                rtnh->rtnh_flags |= RTNH_F_DEAD;
                                dead++;
                        }

                        if (nh->nh_labels && nla_put_labels(skb, RTA_NEWDST,
                                                            nh->nh_labels,
                                                            nh->nh_label))
                                goto nla_put_failure;
                        if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
                            nla_put_via(skb, nh->nh_via_table,
                                        mpls_nh_via(rt, nh),
                                        nh->nh_via_alen))
                                goto nla_put_failure;

                        /* length of rtnetlink header + attributes */
                        rtnh->rtnh_len = nlmsg_get_pos(skb) - (void *)rtnh;
                } endfor_nexthops(rt);

                if (linkdown == rt->rt_nhn)
                        rtm->rtm_flags |= RTNH_F_LINKDOWN;
                if (dead == rt->rt_nhn)
                        rtm->rtm_flags |= RTNH_F_DEAD;

                nla_nest_end(skb, mp);
        }

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

#if IS_ENABLED(CONFIG_INET)
static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh,
                                   struct fib_dump_filter *filter,
                                   struct netlink_callback *cb)
{
        return ip_valid_fib_dump_req(net, nlh, filter, cb);
}
#else
static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh,
                                   struct fib_dump_filter *filter,
                                   struct netlink_callback *cb)
{
        struct netlink_ext_ack *extack = cb->extack;
        struct nlattr *tb[RTA_MAX + 1];
        struct rtmsg *rtm;
        int err, i;

        rtm = nlmsg_payload(nlh, sizeof(*rtm));
        if (!rtm) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid header for FIB dump request");
                return -EINVAL;
        }

        if (rtm->rtm_dst_len || rtm->rtm_src_len  || rtm->rtm_tos   ||
            rtm->rtm_table   || rtm->rtm_scope    || rtm->rtm_type  ||
            rtm->rtm_flags) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for FIB dump request");
                return -EINVAL;
        }

        if (rtm->rtm_protocol) {
                filter->protocol = rtm->rtm_protocol;
                filter->filter_set = 1;
                cb->answer_flags = NLM_F_DUMP_FILTERED;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
                                            rtm_mpls_policy, extack);
        if (err < 0)
                return err;

        for (i = 0; i <= RTA_MAX; ++i) {
                int ifindex;

                if (i == RTA_OIF) {
                        ifindex = nla_get_u32(tb[i]);
                        filter->dev = dev_get_by_index_rcu(net, ifindex);
                        if (!filter->dev)
                                return -ENODEV;
                        filter->filter_set = 1;
                } else if (tb[i]) {
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in dump request");
                        return -EINVAL;
                }
        }

        return 0;
}
#endif

static bool mpls_rt_uses_dev(struct mpls_route *rt,
                             const struct net_device *dev)
{
        if (rt->rt_nhn == 1) {
                struct mpls_nh *nh = rt->rt_nh;

                if (nh->nh_dev == dev)
                        return true;
        } else {
                for_nexthops(rt) {
                        if (nh->nh_dev == dev)
                                return true;
                } endfor_nexthops(rt);
        }

        return false;
}

static int mpls_dump_routes(struct sk_buff *skb, struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        struct mpls_route __rcu **platform_label;
        struct fib_dump_filter filter = {
                .rtnl_held = false,
        };
        unsigned int flags = NLM_F_MULTI;
        size_t platform_labels;
        unsigned int index;
        int err;

        rcu_read_lock();

        if (cb->strict_check) {
                err = mpls_valid_fib_dump_req(net, nlh, &filter, cb);
                if (err < 0)
                        goto err;

                /* for MPLS, there is only 1 table with fixed type and flags.
                 * If either are set in the filter then return nothing.
                 */
                if ((filter.table_id && filter.table_id != RT_TABLE_MAIN) ||
                    (filter.rt_type && filter.rt_type != RTN_UNICAST) ||
                     filter.flags)
                        goto unlock;
        }

        index = cb->args[0];
        if (index < MPLS_LABEL_FIRST_UNRESERVED)
                index = MPLS_LABEL_FIRST_UNRESERVED;

        platform_label = mpls_platform_label_rcu(net, &platform_labels);

        if (filter.filter_set)
                flags |= NLM_F_DUMP_FILTERED;

        for (; index < platform_labels; index++) {
                struct mpls_route *rt;

                rt = rcu_dereference(platform_label[index]);
                if (!rt)
                        continue;

                if ((filter.dev && !mpls_rt_uses_dev(rt, filter.dev)) ||
                    (filter.protocol && rt->rt_protocol != filter.protocol))
                        continue;

                if (mpls_dump_route(skb, NETLINK_CB(cb->skb).portid,
                                    cb->nlh->nlmsg_seq, RTM_NEWROUTE,
                                    index, rt, flags) < 0)
                        break;
        }
        cb->args[0] = index;

unlock:
        rcu_read_unlock();
        return skb->len;

err:
        rcu_read_unlock();
        return err;
}

static inline size_t lfib_nlmsg_size(struct mpls_route *rt)
{
        size_t payload =
                NLMSG_ALIGN(sizeof(struct rtmsg))
                + nla_total_size(4)                        /* RTA_DST */
                + nla_total_size(1);                        /* RTA_TTL_PROPAGATE */

        if (rt->rt_nhn == 1) {
                struct mpls_nh *nh = rt->rt_nh;

                if (nh->nh_dev)
                        payload += nla_total_size(4); /* RTA_OIF */
                if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC) /* RTA_VIA */
                        payload += nla_total_size(2 + nh->nh_via_alen);
                if (nh->nh_labels) /* RTA_NEWDST */
                        payload += nla_total_size(nh->nh_labels * 4);
        } else {
                /* each nexthop is packed in an attribute */
                size_t nhsize = 0;

                for_nexthops(rt) {
                        if (!nh->nh_dev)
                                continue;
                        nhsize += nla_total_size(sizeof(struct rtnexthop));
                        /* RTA_VIA */
                        if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC)
                                nhsize += nla_total_size(2 + nh->nh_via_alen);
                        if (nh->nh_labels)
                                nhsize += nla_total_size(nh->nh_labels * 4);
                } endfor_nexthops(rt);
                /* nested attribute */
                payload += nla_total_size(nhsize);
        }

        return payload;
}

static void rtmsg_lfib(int event, u32 label, struct mpls_route *rt,
                       struct nlmsghdr *nlh, struct net *net, u32 portid,
                       unsigned int nlm_flags)
{
        struct sk_buff *skb;
        u32 seq = nlh ? nlh->nlmsg_seq : 0;
        int err = -ENOBUFS;

        skb = nlmsg_new(lfib_nlmsg_size(rt), GFP_KERNEL);
        if (skb == NULL)
                goto errout;

        err = mpls_dump_route(skb, portid, seq, event, label, rt, nlm_flags);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in lfib_nlmsg_size */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, portid, RTNLGRP_MPLS_ROUTE, nlh, GFP_KERNEL);

        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_MPLS_ROUTE, err);
}

static int mpls_valid_getroute_req(struct sk_buff *skb,
                                   const struct nlmsghdr *nlh,
                                   struct nlattr **tb,
                                   struct netlink_ext_ack *extack)
{
        struct rtmsg *rtm;
        int i, err;

        rtm = nlmsg_payload(nlh, sizeof(*rtm));
        if (!rtm) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid header for get route request");
                return -EINVAL;
        }

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
                                              rtm_mpls_policy, extack);

        if ((rtm->rtm_dst_len && rtm->rtm_dst_len != 20) ||
            rtm->rtm_src_len || rtm->rtm_tos || rtm->rtm_table ||
            rtm->rtm_protocol || rtm->rtm_scope || rtm->rtm_type) {
                NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request");
                return -EINVAL;
        }
        if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Invalid flags for get route request");
                return -EINVAL;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
                                            rtm_mpls_policy, extack);
        if (err)
                return err;

        if ((tb[RTA_DST] || tb[RTA_NEWDST]) && !rtm->rtm_dst_len) {
                NL_SET_ERR_MSG_MOD(extack, "rtm_dst_len must be 20 for MPLS");
                return -EINVAL;
        }

        for (i = 0; i <= RTA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case RTA_DST:
                case RTA_NEWDST:
                        break;
                default:
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int mpls_getroute(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
                         struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        u32 portid = NETLINK_CB(in_skb).portid;
        u32 in_label = LABEL_NOT_SPECIFIED;
        struct nlattr *tb[RTA_MAX + 1];
        struct mpls_route *rt = NULL;
        u32 labels[MAX_NEW_LABELS];
        struct mpls_shim_hdr *hdr;
        unsigned int hdr_size = 0;
        const struct mpls_nh *nh;
        struct net_device *dev;
        struct rtmsg *rtm, *r;
        struct nlmsghdr *nlh;
        struct sk_buff *skb;
        u8 n_labels;
        int err;

        mutex_lock(&net->mpls.platform_mutex);

        err = mpls_valid_getroute_req(in_skb, in_nlh, tb, extack);
        if (err < 0)
                goto errout;

        rtm = nlmsg_data(in_nlh);

        if (tb[RTA_DST]) {
                u8 label_count;

                if (nla_get_labels(tb[RTA_DST], 1, &label_count,
                                   &in_label, extack)) {
                        err = -EINVAL;
                        goto errout;
                }

                if (!mpls_label_ok(net, &in_label, extack)) {
                        err = -EINVAL;
                        goto errout;
                }
        }

        if (in_label < net->mpls.platform_labels)
                rt = mpls_route_input(net, in_label);
        if (!rt) {
                err = -ENETUNREACH;
                goto errout;
        }

        if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
                skb = nlmsg_new(lfib_nlmsg_size(rt), GFP_KERNEL);
                if (!skb) {
                        err = -ENOBUFS;
                        goto errout;
                }

                err = mpls_dump_route(skb, portid, in_nlh->nlmsg_seq,
                                      RTM_NEWROUTE, in_label, rt, 0);
                if (err < 0) {
                        /* -EMSGSIZE implies BUG in lfib_nlmsg_size */
                        WARN_ON(err == -EMSGSIZE);
                        goto errout_free;
                }

                err = rtnl_unicast(skb, net, portid);
                goto errout;
        }

        if (tb[RTA_NEWDST]) {
                if (nla_get_labels(tb[RTA_NEWDST], MAX_NEW_LABELS, &n_labels,
                                   labels, extack) != 0) {
                        err = -EINVAL;
                        goto errout;
                }

                hdr_size = n_labels * sizeof(struct mpls_shim_hdr);
        }

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb) {
                err = -ENOBUFS;
                goto errout;
        }

        skb->protocol = htons(ETH_P_MPLS_UC);

        if (hdr_size) {
                bool bos;
                int i;

                if (skb_cow(skb, hdr_size)) {
                        err = -ENOBUFS;
                        goto errout_free;
                }

                skb_reserve(skb, hdr_size);
                skb_push(skb, hdr_size);
                skb_reset_network_header(skb);

                /* Push new labels */
                hdr = mpls_hdr(skb);
                bos = true;
                for (i = n_labels - 1; i >= 0; i--) {
                        hdr[i] = mpls_entry_encode(labels[i],
                                                   1, 0, bos);
                        bos = false;
                }
        }

        nh = mpls_select_multipath(rt, skb);
        if (!nh) {
                err = -ENETUNREACH;
                goto errout_free;
        }

        if (hdr_size) {
                skb_pull(skb, hdr_size);
                skb_reset_network_header(skb);
        }

        nlh = nlmsg_put(skb, portid, in_nlh->nlmsg_seq,
                        RTM_NEWROUTE, sizeof(*r), 0);
        if (!nlh) {
                err = -EMSGSIZE;
                goto errout_free;
        }

        r = nlmsg_data(nlh);
        r->rtm_family         = AF_MPLS;
        r->rtm_dst_len        = 20;
        r->rtm_src_len        = 0;
        r->rtm_table        = RT_TABLE_MAIN;
        r->rtm_type        = RTN_UNICAST;
        r->rtm_scope        = RT_SCOPE_UNIVERSE;
        r->rtm_protocol = rt->rt_protocol;
        r->rtm_flags        = 0;

        if (nla_put_labels(skb, RTA_DST, 1, &in_label))
                goto nla_put_failure;

        if (nh->nh_labels &&
            nla_put_labels(skb, RTA_NEWDST, nh->nh_labels,
                           nh->nh_label))
                goto nla_put_failure;

        if (nh->nh_via_table != MPLS_NEIGH_TABLE_UNSPEC &&
            nla_put_via(skb, nh->nh_via_table, mpls_nh_via(rt, nh),
                        nh->nh_via_alen))
                goto nla_put_failure;
        dev = nh->nh_dev;
        if (dev && nla_put_u32(skb, RTA_OIF, dev->ifindex))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);

        err = rtnl_unicast(skb, net, portid);
errout:
        mutex_unlock(&net->mpls.platform_mutex);
        return err;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        err = -EMSGSIZE;
errout_free:
        mutex_unlock(&net->mpls.platform_mutex);
        kfree_skb(skb);
        return err;
}

static int resize_platform_label_table(struct net *net, size_t limit)
{
        size_t size = sizeof(struct mpls_route *) * limit;
        size_t old_limit;
        size_t cp_size;
        struct mpls_route __rcu **labels = NULL, **old;
        struct mpls_route *rt0 = NULL, *rt2 = NULL;
        unsigned index;

        if (size) {
                labels = kvzalloc(size, GFP_KERNEL);
                if (!labels)
                        goto nolabels;
        }

        /* In case the predefined labels need to be populated */
        if (limit > MPLS_LABEL_IPV4NULL) {
                struct net_device *lo = net->loopback_dev;

                rt0 = mpls_rt_alloc(1, lo->addr_len, 0);
                if (IS_ERR(rt0))
                        goto nort0;

                rt0->rt_nh->nh_dev = lo;
                netdev_hold(lo, &rt0->rt_nh->nh_dev_tracker, GFP_KERNEL);
                rt0->rt_protocol = RTPROT_KERNEL;
                rt0->rt_payload_type = MPT_IPV4;
                rt0->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
                rt0->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
                rt0->rt_nh->nh_via_alen = lo->addr_len;
                memcpy(__mpls_nh_via(rt0, rt0->rt_nh), lo->dev_addr,
                       lo->addr_len);
        }
        if (limit > MPLS_LABEL_IPV6NULL) {
                struct net_device *lo = net->loopback_dev;

                rt2 = mpls_rt_alloc(1, lo->addr_len, 0);
                if (IS_ERR(rt2))
                        goto nort2;

                rt2->rt_nh->nh_dev = lo;
                netdev_hold(lo, &rt2->rt_nh->nh_dev_tracker, GFP_KERNEL);
                rt2->rt_protocol = RTPROT_KERNEL;
                rt2->rt_payload_type = MPT_IPV6;
                rt2->rt_ttl_propagate = MPLS_TTL_PROP_DEFAULT;
                rt2->rt_nh->nh_via_table = NEIGH_LINK_TABLE;
                rt2->rt_nh->nh_via_alen = lo->addr_len;
                memcpy(__mpls_nh_via(rt2, rt2->rt_nh), lo->dev_addr,
                       lo->addr_len);
        }

        mutex_lock(&net->mpls.platform_mutex);

        /* Remember the original table */
        old = mpls_dereference(net, net->mpls.platform_label);
        old_limit = net->mpls.platform_labels;

        /* Free any labels beyond the new table */
        for (index = limit; index < old_limit; index++)
                mpls_route_update(net, index, NULL, NULL);

        /* Copy over the old labels */
        cp_size = size;
        if (old_limit < limit)
                cp_size = old_limit * sizeof(struct mpls_route *);

        memcpy(labels, old, cp_size);

        /* If needed set the predefined labels */
        if ((old_limit <= MPLS_LABEL_IPV6NULL) &&
            (limit > MPLS_LABEL_IPV6NULL)) {
                RCU_INIT_POINTER(labels[MPLS_LABEL_IPV6NULL], rt2);
                rt2 = NULL;
        }

        if ((old_limit <= MPLS_LABEL_IPV4NULL) &&
            (limit > MPLS_LABEL_IPV4NULL)) {
                RCU_INIT_POINTER(labels[MPLS_LABEL_IPV4NULL], rt0);
                rt0 = NULL;
        }

        /* Update the global pointers */
        local_bh_disable();
        write_seqcount_begin(&net->mpls.platform_label_seq);
        net->mpls.platform_labels = limit;
        rcu_assign_pointer(net->mpls.platform_label, labels);
        write_seqcount_end(&net->mpls.platform_label_seq);
        local_bh_enable();

        mutex_unlock(&net->mpls.platform_mutex);

        mpls_rt_free(rt2);
        mpls_rt_free(rt0);

        if (old) {
                synchronize_rcu();
                kvfree(old);
        }
        return 0;

nort2:
        mpls_rt_free(rt0);
nort0:
        kvfree(labels);
nolabels:
        return -ENOMEM;
}

static int mpls_platform_labels(const struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        struct net *net = table->data;
        int platform_labels = net->mpls.platform_labels;
        int ret;
        struct ctl_table tmp = {
                .procname        = table->procname,
                .data                = &platform_labels,
                .maxlen                = sizeof(int),
                .mode                = table->mode,
                .extra1                = SYSCTL_ZERO,
                .extra2                = &label_limit,
        };

        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);

        if (write && ret == 0)
                ret = resize_platform_label_table(net, platform_labels);

        return ret;
}

#define MPLS_NS_SYSCTL_OFFSET(field)                \
        (&((struct net *)0)->field)

static const struct ctl_table mpls_table[] = {
        {
                .procname        = "platform_labels",
                .data                = NULL,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = mpls_platform_labels,
        },
        {
                .procname        = "ip_ttl_propagate",
                .data                = MPLS_NS_SYSCTL_OFFSET(mpls.ip_ttl_propagate),
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        {
                .procname        = "default_ttl",
                .data                = MPLS_NS_SYSCTL_OFFSET(mpls.default_ttl),
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ONE,
                .extra2                = &ttl_max,
        },
};

static __net_init int mpls_net_init(struct net *net)
{
        size_t table_size = ARRAY_SIZE(mpls_table);
        struct ctl_table *table;
        int i;

        mutex_init(&net->mpls.platform_mutex);
        seqcount_mutex_init(&net->mpls.platform_label_seq, &net->mpls.platform_mutex);

        net->mpls.platform_labels = 0;
        net->mpls.platform_label = NULL;
        net->mpls.ip_ttl_propagate = 1;
        net->mpls.default_ttl = 255;

        table = kmemdup(mpls_table, sizeof(mpls_table), GFP_KERNEL);
        if (table == NULL)
                return -ENOMEM;

        /* Table data contains only offsets relative to the base of
         * the mdev at this point, so make them absolute.
         */
        for (i = 0; i < table_size; i++)
                table[i].data = (char *)net + (uintptr_t)table[i].data;

        net->mpls.ctl = register_net_sysctl_sz(net, "net/mpls", table,
                                               table_size);
        if (net->mpls.ctl == NULL) {
                kfree(table);
                return -ENOMEM;
        }

        return 0;
}

static __net_exit void mpls_net_exit(struct net *net)
{
        struct mpls_route __rcu **platform_label;
        size_t platform_labels;
        const struct ctl_table *table;
        unsigned int index;

        table = net->mpls.ctl->ctl_table_arg;
        unregister_net_sysctl_table(net->mpls.ctl);
        kfree(table);

        /* An rcu grace period has passed since there was a device in
         * the network namespace (and thus the last in flight packet)
         * left this network namespace.  This is because
         * unregister_netdevice_many and netdev_run_todo has completed
         * for each network device that was in this network namespace.
         *
         * As such no additional rcu synchronization is necessary when
         * freeing the platform_label table.
         */
        mutex_lock(&net->mpls.platform_mutex);

        platform_label = mpls_dereference(net, net->mpls.platform_label);
        platform_labels = net->mpls.platform_labels;

        for (index = 0; index < platform_labels; index++) {
                struct mpls_route *rt;

                rt = mpls_dereference(net, platform_label[index]);
                mpls_notify_route(net, index, rt, NULL, NULL);
                mpls_rt_free(rt);
        }

        mutex_unlock(&net->mpls.platform_mutex);

        kvfree(platform_label);
}

static struct pernet_operations mpls_net_ops = {
        .init = mpls_net_init,
        .exit = mpls_net_exit,
};

static struct rtnl_af_ops mpls_af_ops __read_mostly = {
        .family                   = AF_MPLS,
        .fill_stats_af           = mpls_fill_stats_af,
        .get_stats_af_size = mpls_get_stats_af_size,
};

static const struct rtnl_msg_handler mpls_rtnl_msg_handlers[] __initdata_or_module = {
        {THIS_MODULE, PF_MPLS, RTM_NEWROUTE, mpls_rtm_newroute, NULL,
         RTNL_FLAG_DOIT_UNLOCKED},
        {THIS_MODULE, PF_MPLS, RTM_DELROUTE, mpls_rtm_delroute, NULL,
         RTNL_FLAG_DOIT_UNLOCKED},
        {THIS_MODULE, PF_MPLS, RTM_GETROUTE, mpls_getroute, mpls_dump_routes,
         RTNL_FLAG_DOIT_UNLOCKED | RTNL_FLAG_DUMP_UNLOCKED},
        {THIS_MODULE, PF_MPLS, RTM_GETNETCONF,
         mpls_netconf_get_devconf, mpls_netconf_dump_devconf,
         RTNL_FLAG_DOIT_UNLOCKED | RTNL_FLAG_DUMP_UNLOCKED},
};

static int __init mpls_init(void)
{
        int err;

        BUILD_BUG_ON(sizeof(struct mpls_shim_hdr) != 4);

        err = register_pernet_subsys(&mpls_net_ops);
        if (err)
                goto out;

        err = register_netdevice_notifier(&mpls_dev_notifier);
        if (err)
                goto out_unregister_pernet;

        dev_add_pack(&mpls_packet_type);

        err = rtnl_af_register(&mpls_af_ops);
        if (err)
                goto out_unregister_dev_type;

        err = rtnl_register_many(mpls_rtnl_msg_handlers);
        if (err)
                goto out_unregister_rtnl_af;

        err = ipgre_tunnel_encap_add_mpls_ops();
        if (err) {
                pr_err("Can't add mpls over gre tunnel ops\n");
                goto out_unregister_rtnl;
        }

        err = 0;
out:
        return err;

out_unregister_rtnl:
        rtnl_unregister_many(mpls_rtnl_msg_handlers);
out_unregister_rtnl_af:
        rtnl_af_unregister(&mpls_af_ops);
out_unregister_dev_type:
        dev_remove_pack(&mpls_packet_type);
        unregister_netdevice_notifier(&mpls_dev_notifier);
out_unregister_pernet:
        unregister_pernet_subsys(&mpls_net_ops);
        goto out;
}
module_init(mpls_init);

static void __exit mpls_exit(void)
{
        rtnl_unregister_all(PF_MPLS);
        rtnl_af_unregister(&mpls_af_ops);
        dev_remove_pack(&mpls_packet_type);
        unregister_netdevice_notifier(&mpls_dev_notifier);
        unregister_pernet_subsys(&mpls_net_ops);
        ipgre_tunnel_encap_del_mpls_ops();
}
module_exit(mpls_exit);

MODULE_DESCRIPTION("MultiProtocol Label Switching");
MODULE_LICENSE("GPL v2");
MODULE_ALIAS_NETPROTO(PF_MPLS);































































  295 
 1705 








  110 










































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PID_NS_H
#define _LINUX_PID_NS_H

#include <linux/sched.h>
#include <linux/bug.h>
#include <linux/mm.h>
#include <linux/workqueue.h>
#include <linux/threads.h>
#include <linux/nsproxy.h>
#include <linux/ns_common.h>
#include <linux/idr.h>

/* MAX_PID_NS_LEVEL is needed for limiting size of 'struct pid' */
#define MAX_PID_NS_LEVEL 32

struct fs_pin;

#if defined(CONFIG_SYSCTL) && defined(CONFIG_MEMFD_CREATE)
/* modes for vm.memfd_noexec sysctl */
#define MEMFD_NOEXEC_SCOPE_EXEC                        0 /* MFD_EXEC implied if unset */
#define MEMFD_NOEXEC_SCOPE_NOEXEC_SEAL                1 /* MFD_NOEXEC_SEAL implied if unset */
#define MEMFD_NOEXEC_SCOPE_NOEXEC_ENFORCED        2 /* same as 1, except MFD_EXEC rejected */
#endif

struct pid_namespace {
        struct idr idr;
        struct rcu_head rcu;
        unsigned int pid_allocated;
#ifdef CONFIG_SYSCTL
#if defined(CONFIG_MEMFD_CREATE)
        int memfd_noexec_scope;
#endif
        struct ctl_table_set        set;
        struct ctl_table_header *sysctls;
#endif
        struct task_struct *child_reaper;
        struct kmem_cache *pid_cachep;
        unsigned int level;
        int pid_max;
        struct pid_namespace *parent;
#ifdef CONFIG_BSD_PROCESS_ACCT
        struct fs_pin *bacct;
#endif
        struct user_namespace *user_ns;
        struct ucounts *ucounts;
        int reboot;        /* group exit code if this pidns was rebooted */
        struct ns_common ns;
        struct work_struct        work;
} __randomize_layout;

extern struct pid_namespace init_pid_ns;

#define PIDNS_ADDING (1U << 31)

#ifdef CONFIG_PID_NS
static inline struct pid_namespace *to_pid_ns(struct ns_common *ns)
{
        return container_of(ns, struct pid_namespace, ns);
}

static inline struct pid_namespace *get_pid_ns(struct pid_namespace *ns)
{
        ns_ref_inc(ns);
        return ns;
}

#if defined(CONFIG_SYSCTL) && defined(CONFIG_MEMFD_CREATE)
static inline int pidns_memfd_noexec_scope(struct pid_namespace *ns)
{
        int scope = MEMFD_NOEXEC_SCOPE_EXEC;

        for (; ns; ns = ns->parent)
                scope = max(scope, READ_ONCE(ns->memfd_noexec_scope));

        return scope;
}
#else
static inline int pidns_memfd_noexec_scope(struct pid_namespace *ns)
{
        return 0;
}
#endif

extern struct pid_namespace *copy_pid_ns(u64 flags,
        struct user_namespace *user_ns, struct pid_namespace *ns);
extern void zap_pid_ns_processes(struct pid_namespace *pid_ns);
extern int reboot_pid_ns(struct pid_namespace *pid_ns, int cmd);
extern void put_pid_ns(struct pid_namespace *ns);

extern bool pidns_is_ancestor(struct pid_namespace *child,
                              struct pid_namespace *ancestor);

#else /* !CONFIG_PID_NS */
#include <linux/err.h>

static inline struct pid_namespace *get_pid_ns(struct pid_namespace *ns)
{
        return ns;
}

static inline int pidns_memfd_noexec_scope(struct pid_namespace *ns)
{
        return 0;
}

static inline struct pid_namespace *copy_pid_ns(u64 flags,
        struct user_namespace *user_ns, struct pid_namespace *ns)
{
        if (flags & CLONE_NEWPID)
                ns = ERR_PTR(-EINVAL);
        return ns;
}

static inline void put_pid_ns(struct pid_namespace *ns)
{
}

static inline void zap_pid_ns_processes(struct pid_namespace *ns)
{
        BUG();
}

static inline int reboot_pid_ns(struct pid_namespace *pid_ns, int cmd)
{
        return 0;
}

static inline bool pidns_is_ancestor(struct pid_namespace *child,
                                     struct pid_namespace *ancestor)
{
        return false;
}
#endif /* CONFIG_PID_NS */

extern struct pid_namespace *task_active_pid_ns(struct task_struct *tsk);
void pidhash_init(void);
void pid_idr_init(void);
int register_pidns_sysctls(struct pid_namespace *pidns);
void unregister_pidns_sysctls(struct pid_namespace *pidns);

static inline bool task_is_in_init_pid_ns(struct task_struct *tsk)
{
        return task_active_pid_ns(tsk) == &init_pid_ns;
}

#endif /* _LINUX_PID_NS_H */































































   40 
   32 

    8 













1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
// SPDX-License-Identifier: GPL-2.0

#include <linux/netdevice.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>

#include "netdevsim.h"

static int
nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
{
        return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv);
}

static void nsim_taprio_stats(struct tc_taprio_qopt_stats *stats)
{
        stats->window_drops = 0;
        stats->tx_overruns = 0;
}

static int nsim_setup_tc_taprio(struct net_device *dev,
                                struct tc_taprio_qopt_offload *offload)
{
        int err = 0;

        switch (offload->cmd) {
        case TAPRIO_CMD_REPLACE:
        case TAPRIO_CMD_DESTROY:
                break;
        case TAPRIO_CMD_STATS:
                nsim_taprio_stats(&offload->stats);
                break;
        default:
                err = -EOPNOTSUPP;
        }

        return err;
}

static int nsim_setup_tc_ets(struct net_device *dev,
                             struct tc_ets_qopt_offload *offload)
{
        int err = 0;

        switch (offload->command) {
        case TC_ETS_REPLACE:
        case TC_ETS_DESTROY:
                break;
        case TC_ETS_STATS:
                _bstats_update(offload->stats.bstats, 0, 0);
                break;
        default:
                err = -EOPNOTSUPP;
        }

        return err;
}

static LIST_HEAD(nsim_block_cb_list);

int
nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
{
        struct netdevsim *ns = netdev_priv(dev);

        switch (type) {
        case TC_SETUP_QDISC_TAPRIO:
                return nsim_setup_tc_taprio(dev, type_data);
        case TC_SETUP_QDISC_ETS:
                return nsim_setup_tc_ets(dev, type_data);
        case TC_SETUP_BLOCK:
                return flow_block_cb_setup_simple(type_data,
                                                  &nsim_block_cb_list,
                                                  nsim_setup_tc_block_cb,
                                                  ns, ns, true);
        default:
                return -EOPNOTSUPP;
        }
}










































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Common interrupt code for 32 and 64 bit
 */
#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/of.h>
#include <linux/seq_file.h>
#include <linux/smp.h>
#include <linux/ftrace.h>
#include <linux/delay.h>
#include <linux/export.h>
#include <linux/irq.h>
#include <linux/kvm_types.h>

#include <asm/irq_stack.h>
#include <asm/apic.h>
#include <asm/io_apic.h>
#include <asm/irq.h>
#include <asm/mce.h>
#include <asm/hw_irq.h>
#include <asm/desc.h>
#include <asm/traps.h>
#include <asm/thermal.h>
#include <asm/posted_intr.h>
#include <asm/irq_remapping.h>

#if defined(CONFIG_X86_LOCAL_APIC) || defined(CONFIG_X86_THERMAL_VECTOR)
#define CREATE_TRACE_POINTS
#include <asm/trace/irq_vectors.h>
#endif

DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
EXPORT_PER_CPU_SYMBOL(irq_stat);

DEFINE_PER_CPU_CACHE_HOT(u16, __softirq_pending);
EXPORT_PER_CPU_SYMBOL(__softirq_pending);

DEFINE_PER_CPU_CACHE_HOT(struct irq_stack *, hardirq_stack_ptr);

atomic_t irq_err_count;

/*
 * 'what should we do if we get a hw irq event on an illegal vector'.
 * each architecture has to answer this themselves.
 */
void ack_bad_irq(unsigned int irq)
{
        if (printk_ratelimit())
                pr_err("unexpected IRQ trap at vector %02x\n", irq);

        /*
         * Currently unexpected vectors happen only on SMP and APIC.
         * We _must_ ack these because every local APIC has only N
         * irq slots per priority level, and a 'hanging, unacked' IRQ
         * holds up an irq slot - in excessive cases (when multiple
         * unexpected vectors occur) that might lock up the APIC
         * completely.
         * But only ack when the APIC is enabled -AK
         */
        apic_eoi();
}

#define irq_stats(x)                (&per_cpu(irq_stat, x))
/*
 * /proc/interrupts printing for arch specific interrupts
 */
int arch_show_interrupts(struct seq_file *p, int prec)
{
        int j;

        seq_printf(p, "%*s: ", prec, "NMI");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
        seq_puts(p, "  Non-maskable interrupts\n");
#ifdef CONFIG_X86_LOCAL_APIC
        seq_printf(p, "%*s: ", prec, "LOC");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
        seq_puts(p, "  Local timer interrupts\n");

        seq_printf(p, "%*s: ", prec, "SPU");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
        seq_puts(p, "  Spurious interrupts\n");
        seq_printf(p, "%*s: ", prec, "PMI");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
        seq_puts(p, "  Performance monitoring interrupts\n");
        seq_printf(p, "%*s: ", prec, "IWI");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
        seq_puts(p, "  IRQ work interrupts\n");
        seq_printf(p, "%*s: ", prec, "RTR");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
        seq_puts(p, "  APIC ICR read retries\n");
        if (x86_platform_ipi_callback) {
                seq_printf(p, "%*s: ", prec, "PLT");
                for_each_online_cpu(j)
                        seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
                seq_puts(p, "  Platform interrupts\n");
        }
#endif
#ifdef CONFIG_SMP
        seq_printf(p, "%*s: ", prec, "RES");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
        seq_puts(p, "  Rescheduling interrupts\n");
        seq_printf(p, "%*s: ", prec, "CAL");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
        seq_puts(p, "  Function call interrupts\n");
        seq_printf(p, "%*s: ", prec, "TLB");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
        seq_puts(p, "  TLB shootdowns\n");
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
        seq_printf(p, "%*s: ", prec, "TRM");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
        seq_puts(p, "  Thermal event interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
        seq_printf(p, "%*s: ", prec, "THR");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
        seq_puts(p, "  Threshold APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE_AMD
        seq_printf(p, "%*s: ", prec, "DFR");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
        seq_puts(p, "  Deferred Error APIC interrupts\n");
#endif
#ifdef CONFIG_X86_MCE
        seq_printf(p, "%*s: ", prec, "MCE");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
        seq_puts(p, "  Machine check exceptions\n");
        seq_printf(p, "%*s: ", prec, "MCP");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
        seq_puts(p, "  Machine check polls\n");
#endif
#ifdef CONFIG_X86_HV_CALLBACK_VECTOR
        if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
                seq_printf(p, "%*s: ", prec, "HYP");
                for_each_online_cpu(j)
                        seq_printf(p, "%10u ",
                                   irq_stats(j)->irq_hv_callback_count);
                seq_puts(p, "  Hypervisor callback interrupts\n");
        }
#endif
#if IS_ENABLED(CONFIG_HYPERV)
        if (test_bit(HYPERV_REENLIGHTENMENT_VECTOR, system_vectors)) {
                seq_printf(p, "%*s: ", prec, "HRE");
                for_each_online_cpu(j)
                        seq_printf(p, "%10u ",
                                   irq_stats(j)->irq_hv_reenlightenment_count);
                seq_puts(p, "  Hyper-V reenlightenment interrupts\n");
        }
        if (test_bit(HYPERV_STIMER0_VECTOR, system_vectors)) {
                seq_printf(p, "%*s: ", prec, "HVS");
                for_each_online_cpu(j)
                        seq_printf(p, "%10u ",
                                   irq_stats(j)->hyperv_stimer0_count);
                seq_puts(p, "  Hyper-V stimer0 interrupts\n");
        }
#endif
        seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
#if defined(CONFIG_X86_IO_APIC)
        seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
#endif
#if IS_ENABLED(CONFIG_KVM)
        seq_printf(p, "%*s: ", prec, "PIN");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
        seq_puts(p, "  Posted-interrupt notification event\n");

        seq_printf(p, "%*s: ", prec, "NPI");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ",
                           irq_stats(j)->kvm_posted_intr_nested_ipis);
        seq_puts(p, "  Nested posted-interrupt event\n");

        seq_printf(p, "%*s: ", prec, "PIW");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ",
                           irq_stats(j)->kvm_posted_intr_wakeup_ipis);
        seq_puts(p, "  Posted-interrupt wakeup event\n");
#endif
#ifdef CONFIG_GUEST_PERF_EVENTS
        seq_printf(p, "%*s: ", prec, "VPMI");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ",
                           irq_stats(j)->perf_guest_mediated_pmis);
        seq_puts(p, " Perf Guest Mediated PMI\n");
#endif
#ifdef CONFIG_X86_POSTED_MSI
        seq_printf(p, "%*s: ", prec, "PMN");
        for_each_online_cpu(j)
                seq_printf(p, "%10u ",
                           irq_stats(j)->posted_msi_notification_count);
        seq_puts(p, "  Posted MSI notification event\n");
#endif
        return 0;
}

/*
 * /proc/stat helpers
 */
u64 arch_irq_stat_cpu(unsigned int cpu)
{
        u64 sum = irq_stats(cpu)->__nmi_count;

#ifdef CONFIG_X86_LOCAL_APIC
        sum += irq_stats(cpu)->apic_timer_irqs;
        sum += irq_stats(cpu)->irq_spurious_count;
        sum += irq_stats(cpu)->apic_perf_irqs;
        sum += irq_stats(cpu)->apic_irq_work_irqs;
        sum += irq_stats(cpu)->icr_read_retry_count;
        if (x86_platform_ipi_callback)
                sum += irq_stats(cpu)->x86_platform_ipis;
#endif
#ifdef CONFIG_SMP
        sum += irq_stats(cpu)->irq_resched_count;
        sum += irq_stats(cpu)->irq_call_count;
#endif
#ifdef CONFIG_X86_THERMAL_VECTOR
        sum += irq_stats(cpu)->irq_thermal_count;
#endif
#ifdef CONFIG_X86_MCE_THRESHOLD
        sum += irq_stats(cpu)->irq_threshold_count;
#endif
#ifdef CONFIG_X86_HV_CALLBACK_VECTOR
        sum += irq_stats(cpu)->irq_hv_callback_count;
#endif
#if IS_ENABLED(CONFIG_HYPERV)
        sum += irq_stats(cpu)->irq_hv_reenlightenment_count;
        sum += irq_stats(cpu)->hyperv_stimer0_count;
#endif
#ifdef CONFIG_X86_MCE
        sum += per_cpu(mce_exception_count, cpu);
        sum += per_cpu(mce_poll_count, cpu);
#endif
        return sum;
}

u64 arch_irq_stat(void)
{
        u64 sum = atomic_read(&irq_err_count);
        return sum;
}

static __always_inline void handle_irq(struct irq_desc *desc,
                                       struct pt_regs *regs)
{
        if (IS_ENABLED(CONFIG_X86_64))
                generic_handle_irq_desc(desc);
        else
                __handle_irq(desc, regs);
}

static struct irq_desc *reevaluate_vector(int vector)
{
        struct irq_desc *desc = __this_cpu_read(vector_irq[vector]);

        if (!IS_ERR_OR_NULL(desc))
                return desc;

        if (desc == VECTOR_UNUSED)
                pr_emerg_ratelimited("No irq handler for %d.%u\n", smp_processor_id(), vector);
        else
                __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
        return NULL;
}

static __always_inline bool call_irq_handler(int vector, struct pt_regs *regs)
{
        struct irq_desc *desc = __this_cpu_read(vector_irq[vector]);

        if (likely(!IS_ERR_OR_NULL(desc))) {
                handle_irq(desc, regs);
                return true;
        }

        /*
         * Reevaluate with vector_lock held to prevent a race against
         * request_irq() setting up the vector:
         *
         * CPU0                                CPU1
         *                                interrupt is raised in APIC IRR
         *                                but not handled
         * free_irq()
         *   per_cpu(vector_irq, CPU1)[vector] = VECTOR_SHUTDOWN;
         *
         * request_irq()                common_interrupt()
         *                                  d = this_cpu_read(vector_irq[vector]);
         *
         * per_cpu(vector_irq, CPU1)[vector] = desc;
         *
         *                                  if (d == VECTOR_SHUTDOWN)
         *                                    this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
         *
         * This requires that the same vector on the same target CPU is
         * handed out or that a spurious interrupt hits that CPU/vector.
         */
        lock_vector_lock();
        desc = reevaluate_vector(vector);
        unlock_vector_lock();

        if (!desc)
                return false;

        handle_irq(desc, regs);
        return true;
}

/*
 * common_interrupt() handles all normal device IRQ's (the special SMP
 * cross-CPU interrupts have their own entry points).
 */
DEFINE_IDTENTRY_IRQ(common_interrupt)
{
        struct pt_regs *old_regs = set_irq_regs(regs);

        /* entry code tells RCU that we're not quiescent.  Check it. */
        RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");

        if (unlikely(!call_irq_handler(vector, regs)))
                apic_eoi();

        set_irq_regs(old_regs);
}

#ifdef CONFIG_X86_LOCAL_APIC
/* Function pointer for generic interrupt vector handling */
void (*x86_platform_ipi_callback)(void) = NULL;
/*
 * Handler for X86_PLATFORM_IPI_VECTOR.
 */
DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi)
{
        struct pt_regs *old_regs = set_irq_regs(regs);

        apic_eoi();
        trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
        inc_irq_stat(x86_platform_ipis);
        if (x86_platform_ipi_callback)
                x86_platform_ipi_callback();
        trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
        set_irq_regs(old_regs);
}
#endif

#ifdef CONFIG_GUEST_PERF_EVENTS
/*
 * Handler for PERF_GUEST_MEDIATED_PMI_VECTOR.
 */
DEFINE_IDTENTRY_SYSVEC(sysvec_perf_guest_mediated_pmi_handler)
{
         apic_eoi();
         inc_irq_stat(perf_guest_mediated_pmis);
         perf_guest_handle_mediated_pmi();
}
#endif

#if IS_ENABLED(CONFIG_KVM)
static void dummy_handler(void) {}
static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;

void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
{
        if (handler)
                kvm_posted_intr_wakeup_handler = handler;
        else {
                kvm_posted_intr_wakeup_handler = dummy_handler;
                synchronize_rcu();
        }
}
EXPORT_SYMBOL_FOR_KVM(kvm_set_posted_intr_wakeup_handler);

/*
 * Handler for POSTED_INTERRUPT_VECTOR.
 */
DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi)
{
        apic_eoi();
        inc_irq_stat(kvm_posted_intr_ipis);
}

/*
 * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
 */
DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi)
{
        apic_eoi();
        inc_irq_stat(kvm_posted_intr_wakeup_ipis);
        kvm_posted_intr_wakeup_handler();
}

/*
 * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
 */
DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi)
{
        apic_eoi();
        inc_irq_stat(kvm_posted_intr_nested_ipis);
}
#endif

#ifdef CONFIG_X86_POSTED_MSI

/* Posted Interrupt Descriptors for coalesced MSIs to be posted */
DEFINE_PER_CPU_ALIGNED(struct pi_desc, posted_msi_pi_desc);
static DEFINE_PER_CPU_CACHE_HOT(bool, posted_msi_handler_active);

void intel_posted_msi_init(void)
{
        u32 destination, apic_id;

        this_cpu_write(posted_msi_pi_desc.nv, POSTED_MSI_NOTIFICATION_VECTOR);
        /*
         * APIC destination ID is stored in bit 8:15 while in XAPIC mode.
         * VT-d spec. CH 9.11
         */
        apic_id = this_cpu_read(x86_cpu_to_apicid);
        destination = x2apic_enabled() ? apic_id : apic_id << 8;
        this_cpu_write(posted_msi_pi_desc.ndst, destination);
}

void intel_ack_posted_msi_irq(struct irq_data *irqd)
{
        irq_move_irq(irqd);

        /*
         * Handle the rare case that irq_retrigger() raised the actual
         * assigned vector on the target CPU, which means that it was not
         * invoked via the posted MSI handler below. In that case APIC EOI
         * is required as otherwise the ISR entry becomes stale and lower
         * priority interrupts are never going to be delivered after that.
         *
         * If the posted handler invoked the device interrupt handler then
         * the EOI would be premature because it would acknowledge the
         * posted vector.
         */
        if (unlikely(!__this_cpu_read(posted_msi_handler_active)))
                apic_eoi();
}

static __always_inline bool handle_pending_pir(unsigned long *pir, struct pt_regs *regs)
{
        unsigned long pir_copy[NR_PIR_WORDS];
        int vec = FIRST_EXTERNAL_VECTOR;

        if (!pi_harvest_pir(pir, pir_copy))
                return false;

        for_each_set_bit_from(vec, pir_copy, FIRST_SYSTEM_VECTOR)
                call_irq_handler(vec, regs);

        return true;
}

/*
 * Performance data shows that 3 is good enough to harvest 90+% of the
 * benefit on high interrupt rate workloads.
 */
#define MAX_POSTED_MSI_COALESCING_LOOP 3

/*
 * For MSIs that are delivered as posted interrupts, the CPU notifications
 * can be coalesced if the MSIs arrive in high frequency bursts.
 */
DEFINE_IDTENTRY_SYSVEC(sysvec_posted_msi_notification)
{
        struct pi_desc *pid = this_cpu_ptr(&posted_msi_pi_desc);
        struct pt_regs *old_regs = set_irq_regs(regs);

        /* Mark the handler active for intel_ack_posted_msi_irq() */
        __this_cpu_write(posted_msi_handler_active, true);
        inc_irq_stat(posted_msi_notification_count);
        irq_enter();

        /*
         * Loop only MAX_POSTED_MSI_COALESCING_LOOP - 1 times here to take
         * the final handle_pending_pir() invocation after clearing the
         * outstanding notification bit into account.
         */
        for (int i = 1; i < MAX_POSTED_MSI_COALESCING_LOOP; i++) {
                if (!handle_pending_pir(pid->pir, regs))
                        break;
        }

        /*
         * Clear the outstanding notification bit to rearm the notification
         * mechanism.
         */
        pi_clear_on(pid);

        /*
         * Clearing the ON bit can race with a notification. Process the
         * PIR bits one last time so that handling the new interrupts is
         * not delayed until the next notification happens.
         */
        handle_pending_pir(pid->pir, regs);

        apic_eoi();
        irq_exit();
        __this_cpu_write(posted_msi_handler_active, false);
        set_irq_regs(old_regs);
}
#endif /* X86_POSTED_MSI */

#ifdef CONFIG_HOTPLUG_CPU
/* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
void fixup_irqs(void)
{
        unsigned int vector;
        struct irq_desc *desc;
        struct irq_data *data;
        struct irq_chip *chip;

        irq_migrate_all_off_this_cpu();

        /*
         * We can remove mdelay() and then send spurious interrupts to
         * new cpu targets for all the irqs that were handled previously by
         * this cpu. While it works, I have seen spurious interrupt messages
         * (nothing wrong but still...).
         *
         * So for now, retain mdelay(1) and check the IRR and then send those
         * interrupts to new targets as this cpu is already offlined...
         */
        mdelay(1);

        /*
         * We can walk the vector array of this cpu without holding
         * vector_lock because the cpu is already marked !online, so
         * nothing else will touch it.
         */
        for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
                if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
                        continue;

                if (is_vector_pending(vector)) {
                        desc = __this_cpu_read(vector_irq[vector]);

                        raw_spin_lock(&desc->lock);
                        data = irq_desc_get_irq_data(desc);
                        chip = irq_data_get_irq_chip(data);
                        if (chip->irq_retrigger) {
                                chip->irq_retrigger(data);
                                __this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
                        }
                        raw_spin_unlock(&desc->lock);
                }
                if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
                        __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
        }
}
#endif

#ifdef CONFIG_X86_THERMAL_VECTOR
static void smp_thermal_vector(void)
{
        if (x86_thermal_enabled())
                intel_thermal_interrupt();
        else
                pr_err("CPU%d: Unexpected LVT thermal interrupt!\n",
                       smp_processor_id());
}

DEFINE_IDTENTRY_SYSVEC(sysvec_thermal)
{
        trace_thermal_apic_entry(THERMAL_APIC_VECTOR);
        inc_irq_stat(irq_thermal_count);
        smp_thermal_vector();
        trace_thermal_apic_exit(THERMAL_APIC_VECTOR);
        apic_eoi();
}
#endif


















































    5 







    5 

    5 
    5 


    5 


































































































































































































































































































































































   11 






   11 


   11 



   11 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
/*
 * llc_sap.c - driver routines for SAP component.
 *
 * Copyright (c) 1997 by Procom Technology, Inc.
 *                  2001-2003 by Arnaldo Carvalho de Melo <acme@conectiva.com.br>
 *
 * This program can be redistributed or modified under the terms of the
 * GNU General Public License as published by the Free Software Foundation.
 * This program is distributed without any warranty or implied warranty
 * of merchantability or fitness for a particular purpose.
 *
 * See the GNU General Public License for more details.
 */

#include <net/llc.h>
#include <net/llc_if.h>
#include <net/llc_conn.h>
#include <net/llc_pdu.h>
#include <net/llc_sap.h>
#include <net/llc_s_ac.h>
#include <net/llc_s_ev.h>
#include <net/llc_s_st.h>
#include <net/sock.h>
#include <net/tcp_states.h>
#include <linux/llc.h>
#include <linux/slab.h>

static int llc_mac_header_len(unsigned short devtype)
{
        switch (devtype) {
        case ARPHRD_ETHER:
        case ARPHRD_LOOPBACK:
                return sizeof(struct ethhdr);
        }
        return 0;
}

/**
 *        llc_alloc_frame - allocates sk_buff for frame
 *        @sk:  socket to allocate frame to
 *        @dev: network device this skb will be sent over
 *        @type: pdu type to allocate
 *        @data_size: data size to allocate
 *
 *        Allocates an sk_buff for frame and initializes sk_buff fields.
 *        Returns allocated skb or %NULL when out of memory.
 */
struct sk_buff *llc_alloc_frame(struct sock *sk, struct net_device *dev,
                                u8 type, u32 data_size)
{
        int hlen = type == LLC_PDU_TYPE_U ? 3 : 4;
        struct sk_buff *skb;

        hlen += llc_mac_header_len(dev->type);
        skb = alloc_skb(hlen + data_size, GFP_ATOMIC);

        if (skb) {
                skb_reset_mac_header(skb);
                skb_reserve(skb, hlen);
                skb_reset_network_header(skb);
                skb_reset_transport_header(skb);
                skb->protocol = htons(ETH_P_802_2);
                skb->dev      = dev;
                if (sk != NULL)
                        skb_set_owner_w(skb, sk);
        }
        return skb;
}

void llc_save_primitive(struct sock *sk, struct sk_buff *skb, u8 prim)
{
        struct sockaddr_llc *addr;

       /* save primitive for use by the user. */
        addr                  = llc_ui_skb_cb(skb);

        memset(addr, 0, sizeof(*addr));
        addr->sllc_family = sk->sk_family;
        addr->sllc_arphrd = skb->dev->type;
        addr->sllc_test   = prim == LLC_TEST_PRIM;
        addr->sllc_xid    = prim == LLC_XID_PRIM;
        addr->sllc_ua     = prim == LLC_DATAUNIT_PRIM;
        llc_pdu_decode_sa(skb, addr->sllc_mac);
        llc_pdu_decode_ssap(skb, &addr->sllc_sap);
}

/**
 *        llc_sap_rtn_pdu - Informs upper layer on rx of an UI, XID or TEST pdu.
 *        @sap: pointer to SAP
 *        @skb: received pdu
 */
void llc_sap_rtn_pdu(struct llc_sap *sap, struct sk_buff *skb)
{
        struct llc_sap_state_ev *ev = llc_sap_ev(skb);
        struct llc_pdu_un *pdu = llc_pdu_un_hdr(skb);

        switch (LLC_U_PDU_RSP(pdu)) {
        case LLC_1_PDU_CMD_TEST:
                ev->prim = LLC_TEST_PRIM;        break;
        case LLC_1_PDU_CMD_XID:
                ev->prim = LLC_XID_PRIM;        break;
        case LLC_1_PDU_CMD_UI:
                ev->prim = LLC_DATAUNIT_PRIM;        break;
        }
        ev->ind_cfm_flag = LLC_IND;
}

/**
 *        llc_find_sap_trans - finds transition for event
 *        @sap: pointer to SAP
 *        @skb: happened event
 *
 *        This function finds transition that matches with happened event.
 *        Returns the pointer to found transition on success or %NULL for
 *        failure.
 */
static const struct llc_sap_state_trans *llc_find_sap_trans(struct llc_sap *sap,
                                                            struct sk_buff *skb)
{
        int i = 0;
        const struct llc_sap_state_trans *rc = NULL;
        const struct llc_sap_state_trans **next_trans;
        struct llc_sap_state *curr_state = &llc_sap_state_table[sap->state - 1];
        /*
         * Search thru events for this state until list exhausted or until
         * its obvious the event is not valid for the current state
         */
        for (next_trans = curr_state->transitions; next_trans[i]->ev; i++)
                if (!next_trans[i]->ev(sap, skb)) {
                        rc = next_trans[i]; /* got event match; return it */
                        break;
                }
        return rc;
}

/**
 *        llc_exec_sap_trans_actions - execute actions related to event
 *        @sap: pointer to SAP
 *        @trans: pointer to transition that it's actions must be performed
 *        @skb: happened event.
 *
 *        This function executes actions that is related to happened event.
 *        Returns 0 for success and 1 for failure of at least one action.
 */
static int llc_exec_sap_trans_actions(struct llc_sap *sap,
                                      const struct llc_sap_state_trans *trans,
                                      struct sk_buff *skb)
{
        int rc = 0;
        const llc_sap_action_t *next_action = trans->ev_actions;

        for (; next_action && *next_action; next_action++)
                if ((*next_action)(sap, skb))
                        rc = 1;
        return rc;
}

/**
 *        llc_sap_next_state - finds transition, execs actions & change SAP state
 *        @sap: pointer to SAP
 *        @skb: happened event
 *
 *        This function finds transition that matches with happened event, then
 *        executes related actions and finally changes state of SAP. It returns
 *        0 on success and 1 for failure.
 */
static int llc_sap_next_state(struct llc_sap *sap, struct sk_buff *skb)
{
        const struct llc_sap_state_trans *trans;
        int rc = 1;

        if (sap->state > LLC_NR_SAP_STATES)
                goto out;
        trans = llc_find_sap_trans(sap, skb);
        if (!trans)
                goto out;
        /*
         * Got the state to which we next transition; perform the actions
         * associated with this transition before actually transitioning to the
         * next state
         */
        rc = llc_exec_sap_trans_actions(sap, trans, skb);
        if (rc)
                goto out;
        /*
         * Transition SAP to next state if all actions execute successfully
         */
        sap->state = trans->next_state;
out:
        return rc;
}

/**
 *        llc_sap_state_process - sends event to SAP state machine
 *        @sap: sap to use
 *        @skb: pointer to occurred event
 *
 *        After executing actions of the event, upper layer will be indicated
 *        if needed(on receiving an UI frame). sk can be null for the
 *        datalink_proto case.
 *
 *        This function always consumes a reference to the skb.
 */
static void llc_sap_state_process(struct llc_sap *sap, struct sk_buff *skb)
{
        struct llc_sap_state_ev *ev = llc_sap_ev(skb);

        ev->ind_cfm_flag = 0;
        llc_sap_next_state(sap, skb);

        if (ev->ind_cfm_flag == LLC_IND && skb->sk->sk_state != TCP_LISTEN) {
                llc_save_primitive(skb->sk, skb, ev->prim);

                /* queue skb to the user. */
                if (sock_queue_rcv_skb(skb->sk, skb) == 0)
                        return;
        }
        kfree_skb(skb);
}

/**
 *        llc_build_and_send_test_pkt - TEST interface for upper layers.
 *        @sap: sap to use
 *        @skb: packet to send
 *        @dmac: destination mac address
 *        @dsap: destination sap
 *
 *        This function is called when upper layer wants to send a TEST pdu.
 *        Returns 0 for success, 1 otherwise.
 */
void llc_build_and_send_test_pkt(struct llc_sap *sap,
                                 struct sk_buff *skb, u8 *dmac, u8 dsap)
{
        struct llc_sap_state_ev *ev = llc_sap_ev(skb);

        ev->saddr.lsap = sap->laddr.lsap;
        ev->daddr.lsap = dsap;
        memcpy(ev->saddr.mac, skb->dev->dev_addr, IFHWADDRLEN);
        memcpy(ev->daddr.mac, dmac, IFHWADDRLEN);

        ev->type      = LLC_SAP_EV_TYPE_PRIM;
        ev->prim      = LLC_TEST_PRIM;
        ev->prim_type = LLC_PRIM_TYPE_REQ;
        llc_sap_state_process(sap, skb);
}

/**
 *        llc_build_and_send_xid_pkt - XID interface for upper layers
 *        @sap: sap to use
 *        @skb: packet to send
 *        @dmac: destination mac address
 *        @dsap: destination sap
 *
 *        This function is called when upper layer wants to send a XID pdu.
 *        Returns 0 for success, 1 otherwise.
 */
void llc_build_and_send_xid_pkt(struct llc_sap *sap, struct sk_buff *skb,
                                u8 *dmac, u8 dsap)
{
        struct llc_sap_state_ev *ev = llc_sap_ev(skb);

        ev->saddr.lsap = sap->laddr.lsap;
        ev->daddr.lsap = dsap;
        memcpy(ev->saddr.mac, skb->dev->dev_addr, IFHWADDRLEN);
        memcpy(ev->daddr.mac, dmac, IFHWADDRLEN);

        ev->type      = LLC_SAP_EV_TYPE_PRIM;
        ev->prim      = LLC_XID_PRIM;
        ev->prim_type = LLC_PRIM_TYPE_REQ;
        llc_sap_state_process(sap, skb);
}

/**
 *        llc_sap_rcv - sends received pdus to the sap state machine
 *        @sap: current sap component structure.
 *        @skb: received frame.
 *        @sk:  socket to associate to frame
 *
 *        Sends received pdus to the sap state machine.
 */
static void llc_sap_rcv(struct llc_sap *sap, struct sk_buff *skb,
                        struct sock *sk)
{
        struct llc_sap_state_ev *ev = llc_sap_ev(skb);

        ev->type   = LLC_SAP_EV_TYPE_PDU;
        ev->reason = 0;
        skb_orphan(skb);
        sock_hold(sk);
        skb->sk = sk;
        skb->destructor = sock_efree;
        llc_sap_state_process(sap, skb);
}

static inline bool llc_dgram_match(const struct llc_sap *sap,
                                   const struct llc_addr *laddr,
                                   const struct sock *sk,
                                   const struct net *net)
{
     struct llc_sock *llc = llc_sk(sk);

     return sk->sk_type == SOCK_DGRAM &&
             net_eq(sock_net(sk), net) &&
             llc->laddr.lsap == laddr->lsap &&
             ether_addr_equal(llc->laddr.mac, laddr->mac);
}

/**
 *        llc_lookup_dgram - Finds dgram socket for the local sap/mac
 *        @sap: SAP
 *        @laddr: address of local LLC (MAC + SAP)
 *        @net: netns to look up a socket in
 *
 *        Search socket list of the SAP and finds connection using the local
 *        mac, and local sap. Returns pointer for socket found, %NULL otherwise.
 */
static struct sock *llc_lookup_dgram(struct llc_sap *sap,
                                     const struct llc_addr *laddr,
                                     const struct net *net)
{
        struct sock *rc;
        struct hlist_nulls_node *node;
        int slot = llc_sk_laddr_hashfn(sap, laddr);
        struct hlist_nulls_head *laddr_hb = &sap->sk_laddr_hash[slot];

        rcu_read_lock_bh();
again:
        sk_nulls_for_each_rcu(rc, node, laddr_hb) {
                if (llc_dgram_match(sap, laddr, rc, net)) {
                        /* Extra checks required by SLAB_TYPESAFE_BY_RCU */
                        if (unlikely(!refcount_inc_not_zero(&rc->sk_refcnt)))
                                goto again;
                        if (unlikely(llc_sk(rc)->sap != sap ||
                                     !llc_dgram_match(sap, laddr, rc, net))) {
                                sock_put(rc);
                                continue;
                        }
                        goto found;
                }
        }
        rc = NULL;
        /*
         * if the nulls value we got at the end of this lookup is
         * not the expected one, we must restart lookup.
         * We probably met an item that was moved to another chain.
         */
        if (unlikely(get_nulls_value(node) != slot))
                goto again;
found:
        rcu_read_unlock_bh();
        return rc;
}

static inline bool llc_mcast_match(const struct llc_sap *sap,
                                   const struct llc_addr *laddr,
                                   const struct sk_buff *skb,
                                   const struct sock *sk)
{
     struct llc_sock *llc = llc_sk(sk);

     return sk->sk_type == SOCK_DGRAM &&
          llc->laddr.lsap == laddr->lsap &&
          llc->dev == skb->dev;
}

static void llc_do_mcast(struct llc_sap *sap, struct sk_buff *skb,
                         struct sock **stack, int count)
{
        struct sk_buff *skb1;
        int i;

        for (i = 0; i < count; i++) {
                skb1 = skb_clone(skb, GFP_ATOMIC);
                if (!skb1) {
                        sock_put(stack[i]);
                        continue;
                }

                llc_sap_rcv(sap, skb1, stack[i]);
                sock_put(stack[i]);
        }
}

/**
 *         llc_sap_mcast - Deliver multicast PDU's to all matching datagram sockets.
 *        @sap: SAP
 *        @laddr: address of local LLC (MAC + SAP)
 *        @skb: PDU to deliver
 *
 *        Search socket list of the SAP and finds connections with same sap.
 *        Deliver clone to each.
 */
static void llc_sap_mcast(struct llc_sap *sap,
                          const struct llc_addr *laddr,
                          struct sk_buff *skb)
{
        int i = 0;
        struct sock *sk;
        struct sock *stack[256 / sizeof(struct sock *)];
        struct llc_sock *llc;
        struct hlist_head *dev_hb = llc_sk_dev_hash(sap, skb->dev->ifindex);

        spin_lock_bh(&sap->sk_lock);
        hlist_for_each_entry(llc, dev_hb, dev_hash_node) {

                sk = &llc->sk;

                if (!llc_mcast_match(sap, laddr, skb, sk))
                        continue;

                sock_hold(sk);
                if (i < ARRAY_SIZE(stack))
                        stack[i++] = sk;
                else {
                        llc_do_mcast(sap, skb, stack, i);
                        i = 0;
                }
        }
        spin_unlock_bh(&sap->sk_lock);

        llc_do_mcast(sap, skb, stack, i);
}


void llc_sap_handler(struct llc_sap *sap, struct sk_buff *skb)
{
        struct llc_addr laddr;

        llc_pdu_decode_da(skb, laddr.mac);
        llc_pdu_decode_dsap(skb, &laddr.lsap);

        if (is_multicast_ether_addr(laddr.mac)) {
                llc_sap_mcast(sap, &laddr, skb);
                kfree_skb(skb);
        } else {
                struct sock *sk = llc_lookup_dgram(sap, &laddr, dev_net(skb->dev));
                if (sk) {
                        llc_sap_rcv(sap, skb, sk);
                        sock_put(sk);
                } else
                        kfree_skb(skb);
        }
}





























































  124 


  124 

  123 
  115 









    7 


    7 



  113 


  113 









   23 



   23 

    1 




   21 













   22 
   22 







    5 


    5 
    5 


    5 



    3 






    5 
















































































































































































   14 



   14 






   14 

   12 






   14 
   12 

















































    1 


    1 

    1 
    1 














    7 
   31 

   31 





































































































  121 





  120 
  120 
  120 
  120 
  120 
  121 
  121 
  121 
  121 
  121 
  121 
  121 
  121 
  121 
  121 
  121 






















  103 






    7 


















  110 






















  121 





















  121 








  121 


































  110 



  110 









  103 










   12 
  103 






   99 
























  103 













  103 













  103 















   97 
   13 






















  110 









  110 








  110 




  110 





  110 






  110 










































  110 







  110 







  110 




  110 












  110 



  110 




  110 


  109 

  110 


  110 




  110 





  110 









  110 




  110 









  110 














  110 











  110 



































  110 










  110 







  110 





  110 







  110 





   97 











   13 









  110 



  110 







    2 




  109 



  109 

   95 



   13 















   47 
   10 













   51 





    6 


    6 


    6 

















   55 























   30 



   31 


   31 




    3 










    3 






    9 


    9 







    9 






    8 






    8 












    6 






















    6 






   19 





   19 




   19 



   19 
















   19 

















   24 




   24 
   24 







    6 
   18 


   21 





   21 

















   20 








   21 

   17 




   17 











   18 










   15 
   48 


   47 
    3 




   52 



   52 






   27 

   36 



   25 













































   29 
   29 


   29 




   28 


































   41 






















   42 

   42 


   42 






   40 
   39 
   40 









   40 
   40 



   40 








   42 




   41 






   42 








   40 











 1357 





 1263 




  117 

   68 

   42 







   41 










   17 
    6 





   28 








   29 
   29 







   29 











   17 



    4 


   13 




























   17 

   13 
   11 















   18 













































   81 



   82 


   38 
   81 







   86 



   85 

   86 


   86 
   17 





   27 




   27 



   26 

    4 















   31 
   31 


   31 











   59 
   59 








    4 
    4 












































   53 
   52 




















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
// SPDX-License-Identifier: GPL-2.0-only
/*
 * This is the linux wireless configuration interface.
 *
 * Copyright 2006-2010                Johannes Berg <johannes@sipsolutions.net>
 * Copyright 2013-2014  Intel Mobile Communications GmbH
 * Copyright 2015-2017        Intel Deutschland GmbH
 * Copyright (C) 2018-2026 Intel Corporation
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/if.h>
#include <linux/module.h>
#include <linux/err.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/nl80211.h>
#include <linux/debugfs.h>
#include <linux/notifier.h>
#include <linux/device.h>
#include <linux/etherdevice.h>
#include <linux/rtnetlink.h>
#include <linux/sched.h>
#include <net/genetlink.h>
#include <net/cfg80211.h>
#include "nl80211.h"
#include "core.h"
#include "sysfs.h"
#include "debugfs.h"
#include "wext-compat.h"
#include "rdev-ops.h"

/* name for sysfs, %d is appended */
#define PHY_NAME "phy"

/* maximum length of radio debugfs directory name */
#define RADIO_DEBUGFSDIR_MAX_LEN        8

MODULE_AUTHOR("Johannes Berg");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("wireless configuration support");
MODULE_ALIAS_GENL_FAMILY(NL80211_GENL_NAME);

/* RCU-protected (and RTNL for writers) */
LIST_HEAD(cfg80211_rdev_list);
int cfg80211_rdev_list_generation;

/* for debugfs */
static struct dentry *ieee80211_debugfs_dir;

/* for the cleanup, scan and event works */
struct workqueue_struct *cfg80211_wq;

static bool cfg80211_disable_40mhz_24ghz;
module_param(cfg80211_disable_40mhz_24ghz, bool, 0644);
MODULE_PARM_DESC(cfg80211_disable_40mhz_24ghz,
                 "Disable 40MHz support in the 2.4GHz band");

struct cfg80211_registered_device *cfg80211_rdev_by_wiphy_idx(int wiphy_idx)
{
        struct cfg80211_registered_device *result = NULL, *rdev;

        ASSERT_RTNL();

        for_each_rdev(rdev) {
                if (rdev->wiphy_idx == wiphy_idx) {
                        result = rdev;
                        break;
                }
        }

        return result;
}

int get_wiphy_idx(struct wiphy *wiphy)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        return rdev->wiphy_idx;
}

struct wiphy *wiphy_idx_to_wiphy(int wiphy_idx)
{
        struct cfg80211_registered_device *rdev;

        ASSERT_RTNL();

        rdev = cfg80211_rdev_by_wiphy_idx(wiphy_idx);
        if (!rdev)
                return NULL;
        return &rdev->wiphy;
}

static int cfg80211_dev_check_name(struct cfg80211_registered_device *rdev,
                                   const char *newname)
{
        struct cfg80211_registered_device *rdev2;
        int wiphy_idx, taken = -1, digits;

        ASSERT_RTNL();

        if (strlen(newname) > NL80211_WIPHY_NAME_MAXLEN)
                return -EINVAL;

        /* prohibit calling the thing phy%d when %d is not its number */
        sscanf(newname, PHY_NAME "%d%n", &wiphy_idx, &taken);
        if (taken == strlen(newname) && wiphy_idx != rdev->wiphy_idx) {
                /* count number of places needed to print wiphy_idx */
                digits = 1;
                while (wiphy_idx /= 10)
                        digits++;
                /*
                 * deny the name if it is phy<idx> where <idx> is printed
                 * without leading zeroes. taken == strlen(newname) here
                 */
                if (taken == strlen(PHY_NAME) + digits)
                        return -EINVAL;
        }

        /* Ensure another device does not already have this name. */
        for_each_rdev(rdev2)
                if (strcmp(newname, wiphy_name(&rdev2->wiphy)) == 0)
                        return -EINVAL;

        return 0;
}

int cfg80211_dev_rename(struct cfg80211_registered_device *rdev,
                        char *newname)
{
        int result;

        ASSERT_RTNL();
        lockdep_assert_wiphy(&rdev->wiphy);

        /* Ignore nop renames */
        if (strcmp(newname, wiphy_name(&rdev->wiphy)) == 0)
                return 0;

        result = cfg80211_dev_check_name(rdev, newname);
        if (result < 0)
                return result;

        result = device_rename(&rdev->wiphy.dev, newname);
        if (result)
                return result;

        debugfs_change_name(rdev->wiphy.debugfsdir, "%s", newname);

        nl80211_notify_wiphy(rdev, NL80211_CMD_NEW_WIPHY);

        return 0;
}

int cfg80211_switch_netns(struct cfg80211_registered_device *rdev,
                          struct net *net)
{
        struct wireless_dev *wdev;
        int err = 0;

        if (!(rdev->wiphy.flags & WIPHY_FLAG_NETNS_OK))
                return -EOPNOTSUPP;

        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                if (!wdev->netdev)
                        continue;
                wdev->netdev->netns_immutable = false;
                err = dev_change_net_namespace(wdev->netdev, net, "wlan%d");
                if (err)
                        break;
                wdev->netdev->netns_immutable = true;
        }

        if (err) {
                /* failed -- clean up to old netns */
                net = wiphy_net(&rdev->wiphy);

                list_for_each_entry_continue_reverse(wdev,
                                                     &rdev->wiphy.wdev_list,
                                                     list) {
                        if (!wdev->netdev)
                                continue;
                        wdev->netdev->netns_immutable = false;
                        err = dev_change_net_namespace(wdev->netdev, net,
                                                        "wlan%d");
                        WARN_ON(err);
                        wdev->netdev->netns_immutable = true;
                }

                return err;
        }

        guard(wiphy)(&rdev->wiphy);

        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                if (!wdev->netdev)
                        continue;
                nl80211_notify_iface(rdev, wdev, NL80211_CMD_DEL_INTERFACE);
        }

        nl80211_notify_wiphy(rdev, NL80211_CMD_DEL_WIPHY);

        wiphy_net_set(&rdev->wiphy, net);

        err = device_rename(&rdev->wiphy.dev, dev_name(&rdev->wiphy.dev));
        WARN_ON(err);

        nl80211_notify_wiphy(rdev, NL80211_CMD_NEW_WIPHY);

        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                if (!wdev->netdev)
                        continue;
                nl80211_notify_iface(rdev, wdev, NL80211_CMD_NEW_INTERFACE);
        }

        return 0;
}

static void cfg80211_rfkill_poll(struct rfkill *rfkill, void *data)
{
        struct cfg80211_registered_device *rdev = data;

        guard(wiphy)(&rdev->wiphy);

        rdev_rfkill_poll(rdev);
}

void cfg80211_stop_p2p_device(struct cfg80211_registered_device *rdev,
                              struct wireless_dev *wdev)
{
        lockdep_assert_held(&rdev->wiphy.mtx);

        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_P2P_DEVICE))
                return;

        if (!wdev_running(wdev))
                return;

        rdev_stop_p2p_device(rdev, wdev);
        wdev->is_running = false;

        rdev->opencount--;

        if (rdev->scan_req && rdev->scan_req->req.wdev == wdev) {
                if (WARN_ON(!rdev->scan_req->notified &&
                            (!rdev->int_scan_req ||
                             !rdev->int_scan_req->notified)))
                        rdev->scan_req->info.aborted = true;
                ___cfg80211_scan_done(rdev, false);
        }
}

void cfg80211_stop_nan(struct cfg80211_registered_device *rdev,
                       struct wireless_dev *wdev)
{
        struct cfg80211_nan_local_sched empty_sched = {};

        lockdep_assert_held(&rdev->wiphy.mtx);

        if (WARN_ON(wdev->iftype != NL80211_IFTYPE_NAN))
                return;

        if (!wdev_running(wdev))
                return;

        /*
         * If there is a scheduled update pending, mark it as canceled, so the
         * empty schedule will be accepted
         */
        wdev->u.nan.sched_update_pending = false;

        /* Unschedule all */
        cfg80211_nan_set_local_schedule(rdev, wdev, &empty_sched);

        rdev_stop_nan(rdev, wdev);
        wdev->is_running = false;

        eth_zero_addr(wdev->u.nan.cluster_id);

        rdev->opencount--;
}

int cfg80211_nan_set_local_schedule(struct cfg80211_registered_device *rdev,
                                    struct wireless_dev *wdev,
                                    struct cfg80211_nan_local_sched *sched)
{
        int ret;

        lockdep_assert_held(&rdev->wiphy.mtx);

        if (wdev->iftype != NL80211_IFTYPE_NAN || !wdev_running(wdev))
                return -EINVAL;

        if (wdev->u.nan.sched_update_pending)
                return -EBUSY;

        ret = rdev_nan_set_local_sched(rdev, wdev, sched);
        if (ret)
                return ret;

        wdev->u.nan.sched_update_pending = sched->deferred;

        kfree(wdev->u.nan.chandefs);
        wdev->u.nan.chandefs = NULL;
        wdev->u.nan.n_channels = 0;

        if (!sched->n_channels)
                return 0;

        wdev->u.nan.chandefs = kcalloc(sched->n_channels,
                                       sizeof(*wdev->u.nan.chandefs),
                                       GFP_KERNEL);
        if (!wdev->u.nan.chandefs)
                return -ENOMEM;

        for (int i = 0; i < sched->n_channels; i++)
                wdev->u.nan.chandefs[i] = sched->nan_channels[i].chandef;

        wdev->u.nan.n_channels = sched->n_channels;

        return 0;
}

void cfg80211_shutdown_all_interfaces(struct wiphy *wiphy)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        struct wireless_dev *wdev;

        ASSERT_RTNL();

        /*
         * Some netdev interfaces need to be closed before some non-netdev
         * ones, i.e. NAN_DATA interfaces need to be closed before the NAN
         * interface
         */
        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                if (wdev->netdev) {
                        dev_close(wdev->netdev);
                        continue;
                }
        }

        guard(wiphy)(wiphy);

        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                switch (wdev->iftype) {
                case NL80211_IFTYPE_P2P_DEVICE:
                        cfg80211_stop_p2p_device(rdev, wdev);
                        break;
                case NL80211_IFTYPE_NAN:
                        cfg80211_stop_nan(rdev, wdev);
                        break;
                default:
                        break;
                }
        }
}
EXPORT_SYMBOL_GPL(cfg80211_shutdown_all_interfaces);

static int cfg80211_rfkill_set_block(void *data, bool blocked)
{
        struct cfg80211_registered_device *rdev = data;

        if (!blocked)
                return 0;

        rtnl_lock();
        cfg80211_shutdown_all_interfaces(&rdev->wiphy);
        rtnl_unlock();

        return 0;
}

static void cfg80211_rfkill_block_work(struct work_struct *work)
{
        struct cfg80211_registered_device *rdev;

        rdev = container_of(work, struct cfg80211_registered_device,
                            rfkill_block);
        cfg80211_rfkill_set_block(rdev, true);
}

static void cfg80211_event_work(struct work_struct *work)
{
        struct cfg80211_registered_device *rdev;

        rdev = container_of(work, struct cfg80211_registered_device,
                            event_work);

        guard(wiphy)(&rdev->wiphy);

        cfg80211_process_rdev_events(rdev);
}

void cfg80211_destroy_ifaces(struct cfg80211_registered_device *rdev)
{
        struct wireless_dev *wdev, *tmp;

        ASSERT_RTNL();

        list_for_each_entry_safe(wdev, tmp, &rdev->wiphy.wdev_list, list) {
                if (wdev->nl_owner_dead) {
                        cfg80211_close_dependents(rdev, wdev);

                        if (wdev->netdev)
                                dev_close(wdev->netdev);

                        guard(wiphy)(&rdev->wiphy);

                        cfg80211_remove_virtual_intf(rdev, wdev);
                }
        }
}

void cfg80211_close_dependents(struct cfg80211_registered_device *rdev,
                               struct wireless_dev *wdev)
{
        ASSERT_RTNL();

        if (wdev->iftype != NL80211_IFTYPE_NAN)
                return;

        /* Close all NAN DATA interfaces */
        list_for_each_entry(wdev, &rdev->wiphy.wdev_list, list) {
                if (wdev->iftype == NL80211_IFTYPE_NAN_DATA)
                        dev_close(wdev->netdev);
        }
}

static void cfg80211_destroy_iface_wk(struct work_struct *work)
{
        struct cfg80211_registered_device *rdev;

        rdev = container_of(work, struct cfg80211_registered_device,
                            destroy_work);

        rtnl_lock();
        cfg80211_destroy_ifaces(rdev);
        rtnl_unlock();
}

static void cfg80211_sched_scan_stop_wk(struct wiphy *wiphy,
                                        struct wiphy_work *work)
{
        struct cfg80211_registered_device *rdev;
        struct cfg80211_sched_scan_request *req, *tmp;

        rdev = container_of(work, struct cfg80211_registered_device,
                           sched_scan_stop_wk);

        list_for_each_entry_safe(req, tmp, &rdev->sched_scan_req_list, list) {
                if (req->nl_owner_dead)
                        cfg80211_stop_sched_scan_req(rdev, req, false);
        }
}

static void cfg80211_propagate_radar_detect_wk(struct work_struct *work)
{
        struct cfg80211_registered_device *rdev;

        rdev = container_of(work, struct cfg80211_registered_device,
                            propagate_radar_detect_wk);

        rtnl_lock();

        regulatory_propagate_dfs_state(&rdev->wiphy, &rdev->radar_chandef,
                                       NL80211_DFS_UNAVAILABLE,
                                       NL80211_RADAR_DETECTED);

        rtnl_unlock();
}

static void cfg80211_propagate_cac_done_wk(struct work_struct *work)
{
        struct cfg80211_registered_device *rdev;

        rdev = container_of(work, struct cfg80211_registered_device,
                            propagate_cac_done_wk);

        rtnl_lock();

        regulatory_propagate_dfs_state(&rdev->wiphy, &rdev->cac_done_chandef,
                                       NL80211_DFS_AVAILABLE,
                                       NL80211_RADAR_CAC_FINISHED);

        rtnl_unlock();
}

static void cfg80211_wiphy_work(struct work_struct *work)
{
        struct cfg80211_registered_device *rdev;
        struct wiphy_work *wk;

        rdev = container_of(work, struct cfg80211_registered_device, wiphy_work);

        trace_wiphy_work_worker_start(&rdev->wiphy);

        guard(wiphy)(&rdev->wiphy);
        if (rdev->suspended)
                return;

        spin_lock_irq(&rdev->wiphy_work_lock);
        wk = list_first_entry_or_null(&rdev->wiphy_work_list,
                                      struct wiphy_work, entry);
        if (wk) {
                list_del_init(&wk->entry);
                if (!list_empty(&rdev->wiphy_work_list))
                        queue_work(system_dfl_wq, work);
                spin_unlock_irq(&rdev->wiphy_work_lock);

                trace_wiphy_work_run(&rdev->wiphy, wk);
                wk->func(&rdev->wiphy, wk);
        } else {
                spin_unlock_irq(&rdev->wiphy_work_lock);
        }
}

/* exported functions */

struct wiphy *wiphy_new_nm(const struct cfg80211_ops *ops, int sizeof_priv,
                           const char *requested_name)
{
        static atomic_t wiphy_counter = ATOMIC_INIT(0);

        struct cfg80211_registered_device *rdev;
        int alloc_size;

        WARN_ON(ops->add_key && (!ops->del_key || !ops->set_default_key));
        WARN_ON(ops->auth && (!ops->assoc || !ops->deauth || !ops->disassoc));
        WARN_ON(ops->connect && !ops->disconnect);
        WARN_ON(ops->join_ibss && !ops->leave_ibss);
        WARN_ON(ops->add_virtual_intf && !ops->del_virtual_intf);
        WARN_ON(ops->add_station && !ops->del_station);
        WARN_ON(ops->add_mpath && !ops->del_mpath);
        WARN_ON(ops->join_mesh && !ops->leave_mesh);
        WARN_ON(ops->start_p2p_device && !ops->stop_p2p_device);
        WARN_ON(ops->start_ap && !ops->stop_ap);
        WARN_ON(ops->join_ocb && !ops->leave_ocb);
        WARN_ON(ops->suspend && !ops->resume);
        WARN_ON(ops->sched_scan_start && !ops->sched_scan_stop);
        WARN_ON(ops->remain_on_channel && !ops->cancel_remain_on_channel);
        WARN_ON(ops->tdls_channel_switch && !ops->tdls_cancel_channel_switch);
        WARN_ON(ops->add_tx_ts && !ops->del_tx_ts);

        alloc_size = sizeof(*rdev) + sizeof_priv;

        rdev = kzalloc(alloc_size, GFP_KERNEL);
        if (!rdev)
                return NULL;

        rdev->ops = ops;

        rdev->wiphy_idx = atomic_inc_return(&wiphy_counter);

        if (unlikely(rdev->wiphy_idx < 0)) {
                /* ugh, wrapped! */
                atomic_dec(&wiphy_counter);
                kfree(rdev);
                return NULL;
        }

        /* atomic_inc_return makes it start at 1, make it start at 0 */
        rdev->wiphy_idx--;

        /* give it a proper name */
        if (requested_name && requested_name[0]) {
                int rv;

                rtnl_lock();
                rv = cfg80211_dev_check_name(rdev, requested_name);

                if (rv < 0) {
                        rtnl_unlock();
                        goto use_default_name;
                }

                rv = dev_set_name(&rdev->wiphy.dev, "%s", requested_name);
                rtnl_unlock();
                if (rv)
                        goto use_default_name;
        } else {
                int rv;

use_default_name:
                /* NOTE:  This is *probably* safe w/out holding rtnl because of
                 * the restrictions on phy names.  Probably this call could
                 * fail if some other part of the kernel (re)named a device
                 * phyX.  But, might should add some locking and check return
                 * value, and use a different name if this one exists?
                 */
                rv = dev_set_name(&rdev->wiphy.dev, PHY_NAME "%d", rdev->wiphy_idx);
                if (rv < 0) {
                        kfree(rdev);
                        return NULL;
                }
        }

        mutex_init(&rdev->wiphy.mtx);
        INIT_LIST_HEAD(&rdev->wiphy.wdev_list);
        INIT_LIST_HEAD(&rdev->beacon_registrations);
        spin_lock_init(&rdev->beacon_registrations_lock);
        spin_lock_init(&rdev->bss_lock);
        INIT_LIST_HEAD(&rdev->bss_list);
        INIT_LIST_HEAD(&rdev->sched_scan_req_list);
        wiphy_work_init(&rdev->scan_done_wk, __cfg80211_scan_done);
        INIT_DELAYED_WORK(&rdev->dfs_update_channels_wk,
                          cfg80211_dfs_channels_update_work);
#ifdef CONFIG_CFG80211_WEXT
        rdev->wiphy.wext = &cfg80211_wext_handler;
#endif

        device_initialize(&rdev->wiphy.dev);
        rdev->wiphy.dev.class = &ieee80211_class;
        rdev->wiphy.dev.platform_data = rdev;
        device_enable_async_suspend(&rdev->wiphy.dev);

        INIT_WORK(&rdev->destroy_work, cfg80211_destroy_iface_wk);
        wiphy_work_init(&rdev->sched_scan_stop_wk, cfg80211_sched_scan_stop_wk);
        INIT_WORK(&rdev->sched_scan_res_wk, cfg80211_sched_scan_results_wk);
        INIT_WORK(&rdev->propagate_radar_detect_wk,
                  cfg80211_propagate_radar_detect_wk);
        INIT_WORK(&rdev->propagate_cac_done_wk, cfg80211_propagate_cac_done_wk);
        INIT_WORK(&rdev->mgmt_registrations_update_wk,
                  cfg80211_mgmt_registrations_update_wk);
        spin_lock_init(&rdev->mgmt_registrations_lock);
        INIT_WORK(&rdev->wiphy_work, cfg80211_wiphy_work);
        INIT_LIST_HEAD(&rdev->wiphy_work_list);
        spin_lock_init(&rdev->wiphy_work_lock);

#ifdef CONFIG_CFG80211_DEFAULT_PS
        rdev->wiphy.flags |= WIPHY_FLAG_PS_ON_BY_DEFAULT;
#endif

        wiphy_net_set(&rdev->wiphy, &init_net);

        rdev->rfkill_ops.set_block = cfg80211_rfkill_set_block;
        rdev->wiphy.rfkill = rfkill_alloc(dev_name(&rdev->wiphy.dev),
                                          &rdev->wiphy.dev, RFKILL_TYPE_WLAN,
                                          &rdev->rfkill_ops, rdev);

        if (!rdev->wiphy.rfkill) {
                wiphy_free(&rdev->wiphy);
                return NULL;
        }

        INIT_WORK(&rdev->rfkill_block, cfg80211_rfkill_block_work);
        INIT_WORK(&rdev->conn_work, cfg80211_conn_work);
        INIT_WORK(&rdev->event_work, cfg80211_event_work);
        INIT_WORK(&rdev->background_cac_abort_wk,
                  cfg80211_background_cac_abort_wk);
        INIT_DELAYED_WORK(&rdev->background_cac_done_wk,
                          cfg80211_background_cac_done_wk);

        init_waitqueue_head(&rdev->dev_wait);

        /*
         * Initialize wiphy parameters to IEEE 802.11 MIB default values.
         * Fragmentation and RTS threshold are disabled by default with the
         * special -1 value.
         */
        rdev->wiphy.retry_short = 7;
        rdev->wiphy.retry_long = 4;
        rdev->wiphy.frag_threshold = (u32) -1;
        rdev->wiphy.rts_threshold = (u32) -1;
        rdev->wiphy.coverage_class = 0;

        rdev->wiphy.max_num_csa_counters = 1;

        rdev->wiphy.max_sched_scan_plans = 1;
        rdev->wiphy.max_sched_scan_plan_interval = U32_MAX;

        return &rdev->wiphy;
}
EXPORT_SYMBOL(wiphy_new_nm);

static
int wiphy_verify_iface_combinations(struct wiphy *wiphy,
                                    const struct ieee80211_iface_combination *iface_comb,
                                    int n_iface_comb,
                                    bool combined_radio)
{
        const struct ieee80211_iface_combination *c;
        int i, j;

        for (i = 0; i < n_iface_comb; i++) {
                u32 cnt = 0;
                u16 all_iftypes = 0;

                c = &iface_comb[i];

                /*
                 * Combinations with just one interface aren't real,
                 * however we make an exception for DFS.
                 */
                if (WARN_ON((c->max_interfaces < 2) && !c->radar_detect_widths))
                        return -EINVAL;

                /* Need at least one channel */
                if (WARN_ON(!c->num_different_channels))
                        return -EINVAL;

                /* DFS only works on one channel. Avoid this check
                 * for multi-radio global combination, since it hold
                 * the capabilities of all radio combinations.
                 */
                if (!combined_radio &&
                    WARN_ON(c->radar_detect_widths &&
                            c->num_different_channels > 1))
                        return -EINVAL;

                if (WARN_ON(!c->n_limits))
                        return -EINVAL;

                for (j = 0; j < c->n_limits; j++) {
                        u16 types = c->limits[j].types;

                        /* interface types shouldn't overlap */
                        if (WARN_ON(types & all_iftypes))
                                return -EINVAL;
                        all_iftypes |= types;

                        if (WARN_ON(!c->limits[j].max))
                                return -EINVAL;

                        /* Shouldn't list software iftypes in combinations! */
                        if (WARN_ON(wiphy->software_iftypes & types))
                                return -EINVAL;

                        /* Only a single P2P_DEVICE can be allowed, avoid this
                         * check for multi-radio global combination, since it
                         * hold the capabilities of all radio combinations.
                         */
                        if (!combined_radio &&
                            WARN_ON(types & BIT(NL80211_IFTYPE_P2P_DEVICE) &&
                                    c->limits[j].max > 1))
                                return -EINVAL;

                        /* Only a single NAN can be allowed */
                        if (WARN_ON(types & BIT(NL80211_IFTYPE_NAN) &&
                                    c->limits[j].max > 1))
                                return -EINVAL;

                        /*
                         * This isn't well-defined right now. If you have an
                         * IBSS interface, then its beacon interval may change
                         * by joining other networks, and nothing prevents it
                         * from doing that.
                         * So technically we probably shouldn't even allow AP
                         * and IBSS in the same interface, but it seems that
                         * some drivers support that, possibly only with fixed
                         * beacon intervals for IBSS.
                         */
                        if (WARN_ON(types & BIT(NL80211_IFTYPE_ADHOC) &&
                                    c->beacon_int_min_gcd)) {
                                return -EINVAL;
                        }

                        cnt += c->limits[j].max;
                        /*
                         * Don't advertise an unsupported type
                         * in a combination.
                         */
                        if (WARN_ON((wiphy->interface_modes & types) != types))
                                return -EINVAL;
                }

                if (WARN_ON(all_iftypes & BIT(NL80211_IFTYPE_WDS)))
                        return -EINVAL;

                /* You can't even choose that many! */
                if (WARN_ON(cnt < c->max_interfaces))
                        return -EINVAL;
        }

        return 0;
}

static int wiphy_verify_combinations(struct wiphy *wiphy)
{
        int i, ret;
        bool combined_radio = false;

        if (wiphy->n_radio) {
                for (i = 0; i < wiphy->n_radio; i++) {
                        const struct wiphy_radio *radio = &wiphy->radio[i];

                        ret = wiphy_verify_iface_combinations(wiphy,
                                                              radio->iface_combinations,
                                                              radio->n_iface_combinations,
                                                              false);
                        if (ret)
                                return ret;
                }

                combined_radio = true;
        }

        ret = wiphy_verify_iface_combinations(wiphy,
                                              wiphy->iface_combinations,
                                              wiphy->n_iface_combinations,
                                              combined_radio);

        return ret;
}

int wiphy_register(struct wiphy *wiphy)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        int res;
        enum nl80211_band band;
        struct ieee80211_supported_band *sband;
        bool have_band = false;
        int i;
        u16 ifmodes = wiphy->interface_modes;

#ifdef CONFIG_PM
        if (WARN_ON(wiphy->wowlan &&
                    (wiphy->wowlan->flags & WIPHY_WOWLAN_GTK_REKEY_FAILURE) &&
                    !(wiphy->wowlan->flags & WIPHY_WOWLAN_SUPPORTS_GTK_REKEY)))
                return -EINVAL;
        if (WARN_ON(wiphy->wowlan &&
                    !wiphy->wowlan->flags && !wiphy->wowlan->n_patterns &&
                    !wiphy->wowlan->tcp))
                return -EINVAL;
#endif
        if (WARN_ON((wiphy->features & NL80211_FEATURE_TDLS_CHANNEL_SWITCH) &&
                    (!rdev->ops->tdls_channel_switch ||
                     !rdev->ops->tdls_cancel_channel_switch)))
                return -EINVAL;

        if (WARN_ON((wiphy->interface_modes & BIT(NL80211_IFTYPE_NAN)) &&
                    (!rdev->ops->start_nan || !rdev->ops->stop_nan ||
                     !rdev->ops->add_nan_func || !rdev->ops->del_nan_func ||
                     !(wiphy->nan_supported_bands & BIT(NL80211_BAND_2GHZ)))))
                return -EINVAL;

        if (WARN_ON((wiphy->interface_modes & BIT(NL80211_IFTYPE_NAN_DATA)) &&
                    !wiphy->nan_capa.phy.ht.ht_supported))
                return -EINVAL;

        if (WARN_ON(wiphy->interface_modes & BIT(NL80211_IFTYPE_WDS)))
                return -EINVAL;

        if (WARN_ON(wiphy->pmsr_capa && !wiphy->pmsr_capa->ftm.supported))
                return -EINVAL;

        if (wiphy->pmsr_capa && wiphy->pmsr_capa->ftm.supported) {
                if (WARN_ON(!wiphy->pmsr_capa->ftm.asap &&
                            !wiphy->pmsr_capa->ftm.non_asap))
                        return -EINVAL;
                if (WARN_ON(!wiphy->pmsr_capa->ftm.preambles ||
                            !wiphy->pmsr_capa->ftm.bandwidths))
                        return -EINVAL;
                if (WARN_ON(wiphy->pmsr_capa->ftm.preambles &
                                ~(BIT(NL80211_PREAMBLE_LEGACY) |
                                  BIT(NL80211_PREAMBLE_HT) |
                                  BIT(NL80211_PREAMBLE_VHT) |
                                  BIT(NL80211_PREAMBLE_HE) |
                                  BIT(NL80211_PREAMBLE_DMG))))
                        return -EINVAL;
                if (WARN_ON((wiphy->pmsr_capa->ftm.trigger_based ||
                             wiphy->pmsr_capa->ftm.non_trigger_based) &&
                            !(wiphy->pmsr_capa->ftm.preambles &
                              BIT(NL80211_PREAMBLE_HE))))
                        return -EINVAL;
                if (WARN_ON(wiphy->pmsr_capa->ftm.bandwidths &
                                ~(BIT(NL80211_CHAN_WIDTH_20_NOHT) |
                                  BIT(NL80211_CHAN_WIDTH_20) |
                                  BIT(NL80211_CHAN_WIDTH_40) |
                                  BIT(NL80211_CHAN_WIDTH_80) |
                                  BIT(NL80211_CHAN_WIDTH_80P80) |
                                  BIT(NL80211_CHAN_WIDTH_160) |
                                  BIT(NL80211_CHAN_WIDTH_320) |
                                  BIT(NL80211_CHAN_WIDTH_5) |
                                  BIT(NL80211_CHAN_WIDTH_10))))
                        return -EINVAL;
        }

        if (WARN_ON((wiphy->regulatory_flags & REGULATORY_WIPHY_SELF_MANAGED) &&
                    (wiphy->regulatory_flags &
                                        (REGULATORY_CUSTOM_REG |
                                         REGULATORY_STRICT_REG |
                                         REGULATORY_COUNTRY_IE_FOLLOW_POWER |
                                         REGULATORY_COUNTRY_IE_IGNORE))))
                return -EINVAL;

        if (WARN_ON(wiphy->coalesce &&
                    (!wiphy->coalesce->n_rules ||
                     !wiphy->coalesce->n_patterns) &&
                    (!wiphy->coalesce->pattern_min_len ||
                     wiphy->coalesce->pattern_min_len >
                        wiphy->coalesce->pattern_max_len)))
                return -EINVAL;

        if (WARN_ON(wiphy->ap_sme_capa &&
                    !(wiphy->flags & WIPHY_FLAG_HAVE_AP_SME)))
                return -EINVAL;

        if (WARN_ON(wiphy->addresses && !wiphy->n_addresses))
                return -EINVAL;

        if (WARN_ON(wiphy->addresses &&
                    !is_zero_ether_addr(wiphy->perm_addr) &&
                    memcmp(wiphy->perm_addr, wiphy->addresses[0].addr,
                           ETH_ALEN)))
                return -EINVAL;

        if (WARN_ON(wiphy->max_acl_mac_addrs &&
                    (!(wiphy->flags & WIPHY_FLAG_HAVE_AP_SME) ||
                     !rdev->ops->set_mac_acl)))
                return -EINVAL;

        /* assure only valid behaviours are flagged by driver
         * hence subtract 2 as bit 0 is invalid.
         */
        if (WARN_ON(wiphy->bss_select_support &&
                    (wiphy->bss_select_support & ~(BIT(__NL80211_BSS_SELECT_ATTR_AFTER_LAST) - 2))))
                return -EINVAL;

        if (WARN_ON(wiphy_ext_feature_isset(&rdev->wiphy,
                                            NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X) &&
                    (!rdev->ops->set_pmk || !rdev->ops->del_pmk)))
                return -EINVAL;

        if (WARN_ON(!(rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_FW_ROAM) &&
                    rdev->ops->update_connect_params))
                return -EINVAL;

        if (wiphy->addresses)
                memcpy(wiphy->perm_addr, wiphy->addresses[0].addr, ETH_ALEN);

        /* sanity check ifmodes */
        WARN_ON(!ifmodes);
        ifmodes &= ((1 << NUM_NL80211_IFTYPES) - 1) & ~1;
        if (WARN_ON(ifmodes != wiphy->interface_modes))
                wiphy->interface_modes = ifmodes;

        res = wiphy_verify_combinations(wiphy);
        if (res)
                return res;

        /* sanity check supported bands/channels */
        for (band = 0; band < NUM_NL80211_BANDS; band++) {
                const struct ieee80211_sband_iftype_data *iftd;
                u16 types = 0;
                bool have_he = false;

                sband = wiphy->bands[band];
                if (!sband)
                        continue;

                sband->band = band;
                if (WARN_ON(!sband->n_channels))
                        return -EINVAL;
                /*
                 * on 60GHz or sub-1Ghz band, there are no legacy rates, so
                 * n_bitrates is 0
                 */
                if (WARN_ON((band != NL80211_BAND_60GHZ &&
                             band != NL80211_BAND_S1GHZ) &&
                            !sband->n_bitrates))
                        return -EINVAL;

                if (WARN_ON(band == NL80211_BAND_6GHZ &&
                            (sband->ht_cap.ht_supported ||
                             sband->vht_cap.vht_supported)))
                        return -EINVAL;

                /*
                 * Since cfg80211_disable_40mhz_24ghz is global, we can
                 * modify the sband's ht data even if the driver uses a
                 * global structure for that.
                 */
                if (cfg80211_disable_40mhz_24ghz &&
                    band == NL80211_BAND_2GHZ &&
                    sband->ht_cap.ht_supported) {
                        sband->ht_cap.cap &= ~IEEE80211_HT_CAP_SUP_WIDTH_20_40;
                        sband->ht_cap.cap &= ~IEEE80211_HT_CAP_SGI_40;
                }

                /*
                 * Since we use a u32 for rate bitmaps in
                 * ieee80211_get_response_rate, we cannot
                 * have more than 32 legacy rates.
                 */
                if (WARN_ON(sband->n_bitrates > 32))
                        return -EINVAL;

                for (i = 0; i < sband->n_channels; i++) {
                        sband->channels[i].orig_flags =
                                sband->channels[i].flags;
                        sband->channels[i].orig_mag = INT_MAX;
                        sband->channels[i].orig_mpwr =
                                sband->channels[i].max_power;
                        sband->channels[i].band = band;

                        if (WARN_ON(sband->channels[i].freq_offset >= 1000))
                                return -EINVAL;
                }

                for_each_sband_iftype_data(sband, i, iftd) {
                        bool has_ap, has_non_ap;
                        u32 ap_bits = BIT(NL80211_IFTYPE_AP) |
                                      BIT(NL80211_IFTYPE_P2P_GO);

                        if (WARN_ON(!iftd->types_mask))
                                return -EINVAL;
                        if (WARN_ON(types & iftd->types_mask))
                                return -EINVAL;

                        /* at least one piece of information must be present */
                        if (WARN_ON(!iftd->he_cap.has_he))
                                return -EINVAL;

                        types |= iftd->types_mask;

                        if (i == 0)
                                have_he = iftd->he_cap.has_he;
                        else
                                have_he = have_he &&
                                          iftd->he_cap.has_he;

                        has_ap = iftd->types_mask & ap_bits;
                        has_non_ap = iftd->types_mask & ~ap_bits;

                        /*
                         * For EHT 20 MHz STA, the capabilities format differs
                         * but to simplify, don't check 20 MHz but rather check
                         * only if AP and non-AP were mentioned at the same time,
                         * reject if so.
                         */
                        if (WARN_ON(iftd->eht_cap.has_eht &&
                                    has_ap && has_non_ap))
                                return -EINVAL;
                }

                if (WARN_ON(!have_he && band == NL80211_BAND_6GHZ))
                        return -EINVAL;

                have_band = true;
        }

        if (!have_band) {
                WARN_ON(1);
                return -EINVAL;
        }

        for (i = 0; i < rdev->wiphy.n_vendor_commands; i++) {
                /*
                 * Validate we have a policy (can be explicitly set to
                 * VENDOR_CMD_RAW_DATA which is non-NULL) and also that
                 * we have at least one of doit/dumpit.
                 */
                if (WARN_ON(!rdev->wiphy.vendor_commands[i].policy))
                        return -EINVAL;
                if (WARN_ON(!rdev->wiphy.vendor_commands[i].doit &&
                            !rdev->wiphy.vendor_commands[i].dumpit))
                        return -EINVAL;
        }

#ifdef CONFIG_PM
        if (WARN_ON(rdev->wiphy.wowlan && rdev->wiphy.wowlan->n_patterns &&
                    (!rdev->wiphy.wowlan->pattern_min_len ||
                     rdev->wiphy.wowlan->pattern_min_len >
                                rdev->wiphy.wowlan->pattern_max_len)))
                return -EINVAL;
#endif

        if (!wiphy->max_num_akm_suites)
                wiphy->max_num_akm_suites = NL80211_MAX_NR_AKM_SUITES;
        else if (wiphy->max_num_akm_suites < NL80211_MAX_NR_AKM_SUITES ||
                 wiphy->max_num_akm_suites > CFG80211_MAX_NUM_AKM_SUITES)
                return -EINVAL;

        /* Allocate radio configuration space for multi-radio wiphy */
        if (wiphy->n_radio > 0) {
                int idx;

                wiphy->radio_cfg = kzalloc_objs(*wiphy->radio_cfg,
                                                wiphy->n_radio);
                if (!wiphy->radio_cfg)
                        return -ENOMEM;
                /*
                 * Initialize wiphy radio parameters to IEEE 802.11
                 * MIB default values. RTS threshold is disabled by
                 * default with the special -1 value.
                 */
                for (idx = 0; idx < wiphy->n_radio; idx++)
                        wiphy->radio_cfg[idx].rts_threshold = (u32)-1;
        }

        /* check and set up bitrates */
        ieee80211_set_bitrate_flags(wiphy);

        rdev->wiphy.features |= NL80211_FEATURE_SCAN_FLUSH;

        if (rdev->wiphy.bss_param_support & WIPHY_BSS_PARAM_P2P_CTWINDOW)
                rdev->wiphy.features |= NL80211_FEATURE_P2P_GO_CTWIN;
        else if (rdev->wiphy.features & NL80211_FEATURE_P2P_GO_CTWIN)
                rdev->wiphy.bss_param_support |= WIPHY_BSS_PARAM_P2P_CTWINDOW;
        if (rdev->wiphy.bss_param_support & WIPHY_BSS_PARAM_P2P_OPPPS)
                rdev->wiphy.features |= NL80211_FEATURE_P2P_GO_OPPPS;
        else if (rdev->wiphy.features & NL80211_FEATURE_P2P_GO_OPPPS)
                rdev->wiphy.bss_param_support |= WIPHY_BSS_PARAM_P2P_OPPPS;

        rtnl_lock();
        wiphy_lock(&rdev->wiphy);
        res = device_add(&rdev->wiphy.dev);
        if (res) {
                wiphy_unlock(&rdev->wiphy);
                rtnl_unlock();
                return res;
        }

        list_add_rcu(&rdev->list, &cfg80211_rdev_list);
        cfg80211_rdev_list_generation++;

        /* add to debugfs */
        rdev->wiphy.debugfsdir = debugfs_create_dir(wiphy_name(&rdev->wiphy),
                                                    ieee80211_debugfs_dir);
        if (wiphy->n_radio > 0) {
                int idx;
                char radio_name[RADIO_DEBUGFSDIR_MAX_LEN];

                for (idx = 0; idx < wiphy->n_radio; idx++) {
                        scnprintf(radio_name, sizeof(radio_name), "radio%d",
                                  idx);
                        wiphy->radio_cfg[idx].radio_debugfsdir =
                                debugfs_create_dir(radio_name,
                                                   rdev->wiphy.debugfsdir);
                }
        }

        cfg80211_debugfs_rdev_add(rdev);
        nl80211_notify_wiphy(rdev, NL80211_CMD_NEW_WIPHY);
        wiphy_unlock(&rdev->wiphy);

        /* set up regulatory info */
        wiphy_regulatory_register(wiphy);

        if (wiphy->regulatory_flags & REGULATORY_CUSTOM_REG) {
                struct regulatory_request request = {
                        .wiphy_idx = get_wiphy_idx(wiphy),
                        .initiator = NL80211_REGDOM_SET_BY_DRIVER,
                        .alpha2[0] = '9',
                        .alpha2[1] = '9',
                };

                nl80211_send_reg_change_event(&request);
        }

        /* Check that nobody globally advertises any capabilities they do not
         * advertise on all possible interface types.
         */
        if (wiphy->extended_capabilities_len &&
            wiphy->num_iftype_ext_capab &&
            wiphy->iftype_ext_capab) {
                u8 supported_on_all, j;
                const struct wiphy_iftype_ext_capab *capab;

                capab = wiphy->iftype_ext_capab;
                for (j = 0; j < wiphy->extended_capabilities_len; j++) {
                        if (capab[0].extended_capabilities_len > j)
                                supported_on_all =
                                        capab[0].extended_capabilities[j];
                        else
                                supported_on_all = 0x00;
                        for (i = 1; i < wiphy->num_iftype_ext_capab; i++) {
                                if (j >= capab[i].extended_capabilities_len) {
                                        supported_on_all = 0x00;
                                        break;
                                }
                                supported_on_all &=
                                        capab[i].extended_capabilities[j];
                        }
                        if (WARN_ON(wiphy->extended_capabilities[j] &
                                    ~supported_on_all))
                                break;
                }
        }

        rdev->wiphy.registered = true;
        rtnl_unlock();

        res = rfkill_register(rdev->wiphy.rfkill);
        if (res) {
                rfkill_destroy(rdev->wiphy.rfkill);
                rdev->wiphy.rfkill = NULL;
                wiphy_unregister(&rdev->wiphy);
                return res;
        }

        return 0;
}
EXPORT_SYMBOL(wiphy_register);

void wiphy_rfkill_start_polling(struct wiphy *wiphy)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        if (!rdev->ops->rfkill_poll)
                return;
        rdev->rfkill_ops.poll = cfg80211_rfkill_poll;
        rfkill_resume_polling(wiphy->rfkill);
}
EXPORT_SYMBOL(wiphy_rfkill_start_polling);

void cfg80211_process_wiphy_works(struct cfg80211_registered_device *rdev,
                                  struct wiphy_work *end)
{
        unsigned int runaway_limit = 100;
        unsigned long flags;

        lockdep_assert_held(&rdev->wiphy.mtx);

        spin_lock_irqsave(&rdev->wiphy_work_lock, flags);
        while (!list_empty(&rdev->wiphy_work_list)) {
                struct wiphy_work *wk;

                wk = list_first_entry(&rdev->wiphy_work_list,
                                      struct wiphy_work, entry);
                list_del_init(&wk->entry);
                spin_unlock_irqrestore(&rdev->wiphy_work_lock, flags);

                trace_wiphy_work_run(&rdev->wiphy, wk);
                wk->func(&rdev->wiphy, wk);

                spin_lock_irqsave(&rdev->wiphy_work_lock, flags);

                if (wk == end)
                        break;

                if (WARN_ON(--runaway_limit == 0))
                        INIT_LIST_HEAD(&rdev->wiphy_work_list);
        }
        spin_unlock_irqrestore(&rdev->wiphy_work_lock, flags);
}

void wiphy_unregister(struct wiphy *wiphy)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        wait_event(rdev->dev_wait, ({
                int __count;
                wiphy_lock(&rdev->wiphy);
                __count = rdev->opencount;
                wiphy_unlock(&rdev->wiphy);
                __count == 0; }));

        if (rdev->wiphy.rfkill)
                rfkill_unregister(rdev->wiphy.rfkill);

        rtnl_lock();
        wiphy_lock(&rdev->wiphy);
        nl80211_notify_wiphy(rdev, NL80211_CMD_DEL_WIPHY);
        rdev->wiphy.registered = false;

        WARN_ON(!list_empty(&rdev->wiphy.wdev_list));

        /*
         * First remove the hardware from everywhere, this makes
         * it impossible to find from userspace.
         */
        debugfs_remove_recursive(rdev->wiphy.debugfsdir);
        list_del_rcu(&rdev->list);
        synchronize_rcu();

        /*
         * If this device got a regulatory hint tell core its
         * free to listen now to a new shiny device regulatory hint
         */
        wiphy_regulatory_deregister(wiphy);

        cfg80211_rdev_list_generation++;
        device_del(&rdev->wiphy.dev);

#ifdef CONFIG_PM
        if (rdev->wiphy.wowlan_config && rdev->ops->set_wakeup)
                rdev_set_wakeup(rdev, false);
#endif

        /* surely nothing is reachable now, clean up work */
        cfg80211_process_wiphy_works(rdev, NULL);
        wiphy_unlock(&rdev->wiphy);
        rtnl_unlock();

        /* this has nothing to do now but make sure it's gone */
        cancel_work_sync(&rdev->wiphy_work);

        cancel_work_sync(&rdev->rfkill_block);
        cancel_work_sync(&rdev->conn_work);
        flush_work(&rdev->event_work);
        cancel_delayed_work_sync(&rdev->dfs_update_channels_wk);
        cancel_delayed_work_sync(&rdev->background_cac_done_wk);
        flush_work(&rdev->destroy_work);
        flush_work(&rdev->propagate_radar_detect_wk);
        flush_work(&rdev->propagate_cac_done_wk);
        flush_work(&rdev->mgmt_registrations_update_wk);
        flush_work(&rdev->background_cac_abort_wk);

        cfg80211_rdev_free_wowlan(rdev);
        cfg80211_free_coalesce(rdev->coalesce);
        rdev->coalesce = NULL;
}
EXPORT_SYMBOL(wiphy_unregister);

void cfg80211_dev_free(struct cfg80211_registered_device *rdev)
{
        struct cfg80211_internal_bss *scan, *tmp;
        struct cfg80211_beacon_registration *reg, *treg;
        unsigned long flags;

        spin_lock_irqsave(&rdev->wiphy_work_lock, flags);
        WARN_ON(!list_empty(&rdev->wiphy_work_list));
        spin_unlock_irqrestore(&rdev->wiphy_work_lock, flags);
        cancel_work_sync(&rdev->wiphy_work);

        rfkill_destroy(rdev->wiphy.rfkill);
        list_for_each_entry_safe(reg, treg, &rdev->beacon_registrations, list) {
                list_del(&reg->list);
                kfree(reg);
        }
        list_for_each_entry_safe(scan, tmp, &rdev->bss_list, list)
                cfg80211_put_bss(&rdev->wiphy, &scan->pub);
        mutex_destroy(&rdev->wiphy.mtx);

        /*
         * The 'regd' can only be non-NULL if we never finished
         * initializing the wiphy and thus never went through the
         * unregister path - e.g. in failure scenarios. Thus, it
         * cannot have been visible to anyone if non-NULL, so we
         * can just free it here.
         */
        kfree(rcu_dereference_raw(rdev->wiphy.regd));

        kfree(rdev);
}

void wiphy_free(struct wiphy *wiphy)
{
        kfree(wiphy->radio_cfg);
        put_device(&wiphy->dev);
}
EXPORT_SYMBOL(wiphy_free);

void wiphy_rfkill_set_hw_state_reason(struct wiphy *wiphy, bool blocked,
                                      enum rfkill_hard_block_reasons reason)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);

        if (rfkill_set_hw_state_reason(wiphy->rfkill, blocked, reason))
                schedule_work(&rdev->rfkill_block);
}
EXPORT_SYMBOL(wiphy_rfkill_set_hw_state_reason);

static void _cfg80211_unregister_wdev(struct wireless_dev *wdev,
                                      bool unregister_netdev)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy);
        struct cfg80211_cqm_config *cqm_config;
        unsigned int link_id;

        ASSERT_RTNL();
        lockdep_assert_held(&rdev->wiphy.mtx);

        nl80211_notify_iface(rdev, wdev, NL80211_CMD_DEL_INTERFACE);

        wdev->registered = false;

        if (wdev->netdev) {
                sysfs_remove_link(&wdev->netdev->dev.kobj, "phy80211");
                if (unregister_netdev)
                        unregister_netdevice(wdev->netdev);
        }

        list_del_rcu(&wdev->list);
        synchronize_net();
        rdev->devlist_generation++;

        cfg80211_mlme_purge_registrations(wdev);

        switch (wdev->iftype) {
        case NL80211_IFTYPE_P2P_DEVICE:
                cfg80211_stop_p2p_device(rdev, wdev);
                break;
        case NL80211_IFTYPE_NAN:
                cfg80211_stop_nan(rdev, wdev);
                break;
        default:
                break;
        }

#ifdef CONFIG_CFG80211_WEXT
        kfree_sensitive(wdev->wext.keys);
        wdev->wext.keys = NULL;
#endif
        wiphy_work_cancel(wdev->wiphy, &wdev->cqm_rssi_work);
        /* deleted from the list, so can't be found from nl80211 any more */
        cqm_config = rcu_access_pointer(wdev->cqm_config);
        kfree_rcu(cqm_config, rcu_head);
        RCU_INIT_POINTER(wdev->cqm_config, NULL);

        /*
         * Ensure that all events have been processed and
         * freed.
         */
        cfg80211_process_wdev_events(wdev);

        if (wdev->iftype == NL80211_IFTYPE_STATION ||
            wdev->iftype == NL80211_IFTYPE_P2P_CLIENT) {
                for (link_id = 0; link_id < ARRAY_SIZE(wdev->links); link_id++) {
                        struct cfg80211_internal_bss *curbss;

                        curbss = wdev->links[link_id].client.current_bss;

                        if (WARN_ON(curbss)) {
                                cfg80211_unhold_bss(curbss);
                                cfg80211_put_bss(wdev->wiphy, &curbss->pub);
                                wdev->links[link_id].client.current_bss = NULL;
                        }
                }
        }

        wdev->connected = false;
}

void cfg80211_unregister_wdev(struct wireless_dev *wdev)
{
        _cfg80211_unregister_wdev(wdev, true);
}
EXPORT_SYMBOL(cfg80211_unregister_wdev);

static const struct device_type wiphy_type = {
        .name        = "wlan",
};

void cfg80211_update_iface_num(struct cfg80211_registered_device *rdev,
                               enum nl80211_iftype iftype, int num)
{
        lockdep_assert_held(&rdev->wiphy.mtx);

        rdev->num_running_ifaces += num;
        if (iftype == NL80211_IFTYPE_MONITOR)
                rdev->num_running_monitor_ifaces += num;
}

void cfg80211_leave_locked(struct cfg80211_registered_device *rdev,
                           struct wireless_dev *wdev, int link_id)
{
        struct net_device *dev = wdev->netdev;
        struct cfg80211_sched_scan_request *pos, *tmp;

        lockdep_assert_held(&rdev->wiphy.mtx);

        cfg80211_pmsr_wdev_down(wdev);

        cfg80211_stop_radar_detection(wdev);
        cfg80211_stop_background_radar_detection(wdev);

        switch (wdev->iftype) {
        case NL80211_IFTYPE_ADHOC:
                cfg80211_leave_ibss(rdev, dev, true);
                break;
        case NL80211_IFTYPE_P2P_CLIENT:
        case NL80211_IFTYPE_STATION:
                list_for_each_entry_safe(pos, tmp, &rdev->sched_scan_req_list,
                                         list) {
                        if (dev == pos->dev)
                                cfg80211_stop_sched_scan_req(rdev, pos, false);
                }

#ifdef CONFIG_CFG80211_WEXT
                kfree(wdev->wext.ie);
                wdev->wext.ie = NULL;
                wdev->wext.ie_len = 0;
                wdev->wext.connect.auth_type = NL80211_AUTHTYPE_AUTOMATIC;
#endif
                cfg80211_disconnect(rdev, dev,
                                    WLAN_REASON_DEAUTH_LEAVING, true);
                break;
        case NL80211_IFTYPE_MESH_POINT:
                cfg80211_leave_mesh(rdev, dev);
                break;
        case NL80211_IFTYPE_AP:
        case NL80211_IFTYPE_P2P_GO:
                cfg80211_stop_ap(rdev, dev, link_id, true);
                break;
        case NL80211_IFTYPE_OCB:
                cfg80211_leave_ocb(rdev, dev);
                break;
        case NL80211_IFTYPE_P2P_DEVICE:
                cfg80211_stop_p2p_device(rdev, wdev);
                break;
        case NL80211_IFTYPE_NAN:
                cfg80211_stop_nan(rdev, wdev);
                break;
        case NL80211_IFTYPE_AP_VLAN:
        case NL80211_IFTYPE_MONITOR:
        case NL80211_IFTYPE_NAN_DATA:
                /* nothing to do */
                break;
        case NL80211_IFTYPE_UNSPECIFIED:
        case NL80211_IFTYPE_WDS:
        case NUM_NL80211_IFTYPES:
                /* invalid */
                break;
        }
}

void cfg80211_leave(struct cfg80211_registered_device *rdev,
                    struct wireless_dev *wdev, int link_id)
{
        ASSERT_RTNL();

        /* NAN_DATA interfaces must be closed before stopping NAN */
        cfg80211_close_dependents(rdev, wdev);

        guard(wiphy)(&rdev->wiphy);

        cfg80211_leave_locked(rdev, wdev, link_id);
}

void cfg80211_stop_link(struct wiphy *wiphy, struct wireless_dev *wdev,
                        int link_id, gfp_t gfp)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        struct cfg80211_event *ev;
        unsigned long flags;

        /* Only AP/GO interfaces may have a specific link_id */
        if (WARN_ON_ONCE(link_id != -1 &&
                         wdev->iftype != NL80211_IFTYPE_AP &&
                         wdev->iftype != NL80211_IFTYPE_P2P_GO))
                link_id = -1;

        trace_cfg80211_stop_link(wiphy, wdev, link_id);

        if (wdev->iftype == NL80211_IFTYPE_NAN)
                return;

        ev = kzalloc_obj(*ev, gfp);
        if (!ev)
                return;

        ev->type = EVENT_STOPPED;
        ev->link_id = link_id;

        spin_lock_irqsave(&wdev->event_lock, flags);
        list_add_tail(&ev->list, &wdev->event_list);
        spin_unlock_irqrestore(&wdev->event_lock, flags);
        queue_work(cfg80211_wq, &rdev->event_work);
}
EXPORT_SYMBOL(cfg80211_stop_link);

void cfg80211_init_wdev(struct wireless_dev *wdev)
{
        INIT_LIST_HEAD(&wdev->event_list);
        spin_lock_init(&wdev->event_lock);
        INIT_LIST_HEAD(&wdev->mgmt_registrations);
        INIT_LIST_HEAD(&wdev->pmsr_list);
        spin_lock_init(&wdev->pmsr_lock);
        INIT_WORK(&wdev->pmsr_free_wk, cfg80211_pmsr_free_wk);

#ifdef CONFIG_CFG80211_WEXT
        wdev->wext.default_key = -1;
        wdev->wext.default_mgmt_key = -1;
        wdev->wext.connect.auth_type = NL80211_AUTHTYPE_AUTOMATIC;
#endif

        wiphy_work_init(&wdev->cqm_rssi_work, cfg80211_cqm_rssi_notify_work);

        if (wdev->wiphy->flags & WIPHY_FLAG_PS_ON_BY_DEFAULT)
                wdev->ps = true;
        else
                wdev->ps = false;
        /* allow mac80211 to determine the timeout */
        wdev->ps_timeout = -1;

        wdev->radio_mask = BIT(wdev->wiphy->n_radio) - 1;

        if ((wdev->iftype == NL80211_IFTYPE_STATION ||
             wdev->iftype == NL80211_IFTYPE_P2P_CLIENT ||
             wdev->iftype == NL80211_IFTYPE_ADHOC) && !wdev->use_4addr)
                wdev->netdev->priv_flags |= IFF_DONT_BRIDGE;

        INIT_WORK(&wdev->disconnect_wk, cfg80211_autodisconnect_wk);
}

void cfg80211_register_wdev(struct cfg80211_registered_device *rdev,
                            struct wireless_dev *wdev)
{
        ASSERT_RTNL();
        lockdep_assert_held(&rdev->wiphy.mtx);

        /*
         * We get here also when the interface changes network namespaces,
         * as it's registered into the new one, but we don't want it to
         * change ID in that case. Checking if the ID is already assigned
         * works, because 0 isn't considered a valid ID and the memory is
         * 0-initialized.
         */
        if (!wdev->identifier)
                wdev->identifier = ++rdev->wdev_id;
        list_add_rcu(&wdev->list, &rdev->wiphy.wdev_list);
        rdev->devlist_generation++;
        wdev->registered = true;

        if (wdev->netdev &&
            sysfs_create_link(&wdev->netdev->dev.kobj, &rdev->wiphy.dev.kobj,
                              "phy80211"))
                pr_err("failed to add phy80211 symlink to netdev!\n");

        nl80211_notify_iface(rdev, wdev, NL80211_CMD_NEW_INTERFACE);
}

int cfg80211_register_netdevice(struct net_device *dev)
{
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev;
        int ret;

        ASSERT_RTNL();

        if (WARN_ON(!wdev))
                return -EINVAL;

        rdev = wiphy_to_rdev(wdev->wiphy);

        lockdep_assert_held(&rdev->wiphy.mtx);

        /* we'll take care of this */
        wdev->registered = true;
        wdev->registering = true;
        ret = register_netdevice(dev);
        if (ret)
                goto out;

        cfg80211_register_wdev(rdev, wdev);
        ret = 0;
out:
        wdev->registering = false;
        if (ret)
                wdev->registered = false;
        return ret;
}
EXPORT_SYMBOL(cfg80211_register_netdevice);

static int cfg80211_netdev_notifier_call(struct notifier_block *nb,
                                         unsigned long state, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct wireless_dev *wdev = dev->ieee80211_ptr;
        struct cfg80211_registered_device *rdev;
        struct cfg80211_sched_scan_request *pos, *tmp;

        if (!wdev)
                return NOTIFY_DONE;

        rdev = wiphy_to_rdev(wdev->wiphy);

        WARN_ON(wdev->iftype == NL80211_IFTYPE_UNSPECIFIED);

        switch (state) {
        case NETDEV_POST_INIT:
                SET_NETDEV_DEVTYPE(dev, &wiphy_type);
                wdev->netdev = dev;
                /* can only change netns with wiphy */
                dev->netns_immutable = true;

                cfg80211_init_wdev(wdev);
                break;
        case NETDEV_REGISTER:
                if (!wdev->registered) {
                        guard(wiphy)(&rdev->wiphy);

                        cfg80211_register_wdev(rdev, wdev);
                }
                break;
        case NETDEV_UNREGISTER:
                /*
                 * It is possible to get NETDEV_UNREGISTER multiple times,
                 * so check wdev->registered.
                 */
                if (wdev->registered && !wdev->registering) {
                        guard(wiphy)(&rdev->wiphy);

                        _cfg80211_unregister_wdev(wdev, false);
                }
                break;
        case NETDEV_GOING_DOWN:
                cfg80211_leave(rdev, wdev, -1);
                scoped_guard(wiphy, &rdev->wiphy)
                        cfg80211_remove_links(wdev);
                /* since we just did cfg80211_leave() nothing to do there */
                cancel_work_sync(&wdev->disconnect_wk);
                cancel_work_sync(&wdev->pmsr_free_wk);
                break;
        case NETDEV_DOWN:
                wiphy_lock(&rdev->wiphy);
                cfg80211_update_iface_num(rdev, wdev->iftype, -1);
                if (rdev->scan_req && rdev->scan_req->req.wdev == wdev) {
                        if (WARN_ON(!rdev->scan_req->notified &&
                                    (!rdev->int_scan_req ||
                                     !rdev->int_scan_req->notified)))
                                rdev->scan_req->info.aborted = true;
                        ___cfg80211_scan_done(rdev, false);
                }

                list_for_each_entry_safe(pos, tmp,
                                         &rdev->sched_scan_req_list, list) {
                        if (WARN_ON(pos->dev == wdev->netdev))
                                cfg80211_stop_sched_scan_req(rdev, pos, false);
                }

                rdev->opencount--;
                wiphy_unlock(&rdev->wiphy);
                wake_up(&rdev->dev_wait);
                break;
        case NETDEV_UP:
                wiphy_lock(&rdev->wiphy);
                cfg80211_update_iface_num(rdev, wdev->iftype, 1);
                switch (wdev->iftype) {
#ifdef CONFIG_CFG80211_WEXT
                case NL80211_IFTYPE_ADHOC:
                        cfg80211_ibss_wext_join(rdev, wdev);
                        break;
                case NL80211_IFTYPE_STATION:
                        cfg80211_mgd_wext_connect(rdev, wdev);
                        break;
#endif
#ifdef CONFIG_MAC80211_MESH
                case NL80211_IFTYPE_MESH_POINT:
                        {
                                /* backward compat code... */
                                struct mesh_setup setup;
                                memcpy(&setup, &default_mesh_setup,
                                                sizeof(setup));
                                 /* back compat only needed for mesh_id */
                                setup.mesh_id = wdev->u.mesh.id;
                                setup.mesh_id_len = wdev->u.mesh.id_up_len;
                                if (wdev->u.mesh.id_up_len)
                                        __cfg80211_join_mesh(rdev, dev,
                                                        &setup,
                                                        &default_mesh_config);
                                break;
                        }
#endif
                default:
                        break;
                }
                rdev->opencount++;

                /*
                 * Configure power management to the driver here so that its
                 * correctly set also after interface type changes etc.
                 */
                if ((wdev->iftype == NL80211_IFTYPE_STATION ||
                     wdev->iftype == NL80211_IFTYPE_P2P_CLIENT) &&
                    rdev->ops->set_power_mgmt &&
                    rdev_set_power_mgmt(rdev, dev, wdev->ps,
                                        wdev->ps_timeout)) {
                        /* assume this means it's off */
                        wdev->ps = false;
                }
                wiphy_unlock(&rdev->wiphy);
                break;
        case NETDEV_PRE_UP:
                if (!cfg80211_iftype_allowed(wdev->wiphy, wdev->iftype,
                                             wdev->use_4addr, 0))
                        return notifier_from_errno(-EOPNOTSUPP);

                if (rfkill_blocked(rdev->wiphy.rfkill))
                        return notifier_from_errno(-ERFKILL);

                /* NAN_DATA interfaces require a running NAN interface */
                if (wdev->iftype == NL80211_IFTYPE_NAN_DATA) {
                        struct wireless_dev *iter;
                        bool nan_started = false;

                        list_for_each_entry(iter, &rdev->wiphy.wdev_list, list) {
                                if (iter->iftype == NL80211_IFTYPE_NAN &&
                                    wdev_running(iter)) {
                                        nan_started = true;
                                        break;
                                }
                        }

                        if (!nan_started)
                                return notifier_from_errno(-ENOLINK);
                }
                break;
        default:
                return NOTIFY_DONE;
        }

        wireless_nlevent_flush();

        return NOTIFY_OK;
}

static struct notifier_block cfg80211_netdev_notifier = {
        .notifier_call = cfg80211_netdev_notifier_call,
};

static void __net_exit cfg80211_pernet_exit(struct net *net)
{
        struct cfg80211_registered_device *rdev;

        rtnl_lock();
        for_each_rdev(rdev) {
                if (net_eq(wiphy_net(&rdev->wiphy), net))
                        WARN_ON(cfg80211_switch_netns(rdev, &init_net));
        }
        rtnl_unlock();
}

static struct pernet_operations cfg80211_pernet_ops = {
        .exit = cfg80211_pernet_exit,
};

void wiphy_work_queue(struct wiphy *wiphy, struct wiphy_work *work)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        unsigned long flags;

        trace_wiphy_work_queue(wiphy, work);

        spin_lock_irqsave(&rdev->wiphy_work_lock, flags);
        if (list_empty(&work->entry))
                list_add_tail(&work->entry, &rdev->wiphy_work_list);
        spin_unlock_irqrestore(&rdev->wiphy_work_lock, flags);

        queue_work(system_dfl_wq, &rdev->wiphy_work);
}
EXPORT_SYMBOL_GPL(wiphy_work_queue);

void wiphy_work_cancel(struct wiphy *wiphy, struct wiphy_work *work)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        unsigned long flags;

        lockdep_assert_held(&wiphy->mtx);

        trace_wiphy_work_cancel(wiphy, work);

        spin_lock_irqsave(&rdev->wiphy_work_lock, flags);
        if (!list_empty(&work->entry))
                list_del_init(&work->entry);
        spin_unlock_irqrestore(&rdev->wiphy_work_lock, flags);
}
EXPORT_SYMBOL_GPL(wiphy_work_cancel);

void wiphy_work_flush(struct wiphy *wiphy, struct wiphy_work *work)
{
        struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
        unsigned long flags;
        bool run;

        trace_wiphy_work_flush(wiphy, work);

        spin_lock_irqsave(&rdev->wiphy_work_lock, flags);
        run = !work || !list_empty(&work->entry);
        spin_unlock_irqrestore(&rdev->wiphy_work_lock, flags);

        if (run)
                cfg80211_process_wiphy_works(rdev, work);
}
EXPORT_SYMBOL_GPL(wiphy_work_flush);

void wiphy_delayed_work_timer(struct timer_list *t)
{
        struct wiphy_delayed_work *dwork = timer_container_of(dwork, t, timer);

        wiphy_work_queue(dwork->wiphy, &dwork->work);
}
EXPORT_SYMBOL(wiphy_delayed_work_timer);

void wiphy_delayed_work_queue(struct wiphy *wiphy,
                              struct wiphy_delayed_work *dwork,
                              unsigned long delay)
{
        trace_wiphy_delayed_work_queue(wiphy, &dwork->work, delay);

        if (!delay) {
                timer_delete(&dwork->timer);
                wiphy_work_queue(wiphy, &dwork->work);
                return;
        }

        dwork->wiphy = wiphy;
        mod_timer(&dwork->timer, jiffies + delay);
}
EXPORT_SYMBOL_GPL(wiphy_delayed_work_queue);

void wiphy_delayed_work_cancel(struct wiphy *wiphy,
                               struct wiphy_delayed_work *dwork)
{
        lockdep_assert_held(&wiphy->mtx);

        timer_delete_sync(&dwork->timer);
        wiphy_work_cancel(wiphy, &dwork->work);
}
EXPORT_SYMBOL_GPL(wiphy_delayed_work_cancel);

void wiphy_delayed_work_flush(struct wiphy *wiphy,
                              struct wiphy_delayed_work *dwork)
{
        lockdep_assert_held(&wiphy->mtx);

        timer_delete_sync(&dwork->timer);
        wiphy_work_flush(wiphy, &dwork->work);
}
EXPORT_SYMBOL_GPL(wiphy_delayed_work_flush);

bool wiphy_delayed_work_pending(struct wiphy *wiphy,
                                struct wiphy_delayed_work *dwork)
{
        return timer_pending(&dwork->timer);
}
EXPORT_SYMBOL_GPL(wiphy_delayed_work_pending);

enum hrtimer_restart wiphy_hrtimer_work_timer(struct hrtimer *t)
{
        struct wiphy_hrtimer_work *hrwork =
                container_of(t, struct wiphy_hrtimer_work, timer);

        wiphy_work_queue(hrwork->wiphy, &hrwork->work);

        return HRTIMER_NORESTART;
}
EXPORT_SYMBOL_GPL(wiphy_hrtimer_work_timer);

void wiphy_hrtimer_work_queue(struct wiphy *wiphy,
                              struct wiphy_hrtimer_work *hrwork,
                              ktime_t delay)
{
        trace_wiphy_hrtimer_work_queue(wiphy, &hrwork->work, delay);

        if (!delay) {
                hrtimer_cancel(&hrwork->timer);
                wiphy_work_queue(wiphy, &hrwork->work);
                return;
        }

        hrwork->wiphy = wiphy;
        hrtimer_start_range_ns(&hrwork->timer, delay,
                               1000 * NSEC_PER_USEC, HRTIMER_MODE_REL);
}
EXPORT_SYMBOL_GPL(wiphy_hrtimer_work_queue);

void wiphy_hrtimer_work_cancel(struct wiphy *wiphy,
                               struct wiphy_hrtimer_work *hrwork)
{
        lockdep_assert_held(&wiphy->mtx);

        hrtimer_cancel(&hrwork->timer);
        wiphy_work_cancel(wiphy, &hrwork->work);
}
EXPORT_SYMBOL_GPL(wiphy_hrtimer_work_cancel);

void wiphy_hrtimer_work_flush(struct wiphy *wiphy,
                              struct wiphy_hrtimer_work *hrwork)
{
        lockdep_assert_held(&wiphy->mtx);

        hrtimer_cancel(&hrwork->timer);
        wiphy_work_flush(wiphy, &hrwork->work);
}
EXPORT_SYMBOL_GPL(wiphy_hrtimer_work_flush);

bool wiphy_hrtimer_work_pending(struct wiphy *wiphy,
                                struct wiphy_hrtimer_work *hrwork)
{
        return hrtimer_is_queued(&hrwork->timer);
}
EXPORT_SYMBOL_GPL(wiphy_hrtimer_work_pending);

static int __init cfg80211_init(void)
{
        int err;

        err = register_pernet_device(&cfg80211_pernet_ops);
        if (err)
                goto out_fail_pernet;

        err = wiphy_sysfs_init();
        if (err)
                goto out_fail_sysfs;

        err = register_netdevice_notifier(&cfg80211_netdev_notifier);
        if (err)
                goto out_fail_notifier;

        err = nl80211_init();
        if (err)
                goto out_fail_nl80211;

        ieee80211_debugfs_dir = debugfs_create_dir("ieee80211", NULL);

        err = regulatory_init();
        if (err)
                goto out_fail_reg;

        cfg80211_wq = alloc_ordered_workqueue("cfg80211", WQ_MEM_RECLAIM);
        if (!cfg80211_wq) {
                err = -ENOMEM;
                goto out_fail_wq;
        }

        return 0;

out_fail_wq:
        regulatory_exit();
out_fail_reg:
        debugfs_remove(ieee80211_debugfs_dir);
        nl80211_exit();
out_fail_nl80211:
        unregister_netdevice_notifier(&cfg80211_netdev_notifier);
out_fail_notifier:
        wiphy_sysfs_exit();
out_fail_sysfs:
        unregister_pernet_device(&cfg80211_pernet_ops);
out_fail_pernet:
        return err;
}
fs_initcall(cfg80211_init);

static void __exit cfg80211_exit(void)
{
        debugfs_remove(ieee80211_debugfs_dir);
        nl80211_exit();
        unregister_netdevice_notifier(&cfg80211_netdev_notifier);
        wiphy_sysfs_exit();
        regulatory_exit();
        unregister_pernet_device(&cfg80211_pernet_ops);
        destroy_workqueue(cfg80211_wq);
}
module_exit(cfg80211_exit);











































































    1 





    1 





    1 
    1 





    1 






    1 










    1 
    1 


















    1 





    1 
    1 
    1 

































    1 








































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _MM_SWAP_H
#define _MM_SWAP_H

#include <linux/atomic.h> /* for atomic_long_t */
struct mempolicy;
struct swap_iocb;

extern int page_cluster;

#ifdef CONFIG_THP_SWAP
#define SWAPFILE_CLUSTER        HPAGE_PMD_NR
#define swap_entry_order(order)        (order)
#else
#define SWAPFILE_CLUSTER        256
#define swap_entry_order(order)        0
#endif

extern struct swap_info_struct *swap_info[];

/*
 * We use this to track usage of a cluster. A cluster is a block of swap disk
 * space with SWAPFILE_CLUSTER pages long and naturally aligns in disk. All
 * free clusters are organized into a list. We fetch an entry from the list to
 * get a free cluster.
 *
 * The flags field determines if a cluster is free. This is
 * protected by cluster lock.
 */
struct swap_cluster_info {
        spinlock_t lock;        /*
                                 * Protect swap_cluster_info fields
                                 * other than list, and swap_info_struct->swap_map
                                 * elements corresponding to the swap cluster.
                                 */
        u16 count;
        u8 flags;
        u8 order;
        atomic_long_t __rcu *table;        /* Swap table entries, see mm/swap_table.h */
        unsigned int *extend_table;        /* For large swap count, protected by ci->lock */
        struct list_head list;
};

/* All on-list cluster must have a non-zero flag. */
enum swap_cluster_flags {
        CLUSTER_FLAG_NONE = 0, /* For temporary off-list cluster */
        CLUSTER_FLAG_FREE,
        CLUSTER_FLAG_NONFULL,
        CLUSTER_FLAG_FRAG,
        /* Clusters with flags above are allocatable */
        CLUSTER_FLAG_USABLE = CLUSTER_FLAG_FRAG,
        CLUSTER_FLAG_FULL,
        CLUSTER_FLAG_DISCARD,
        CLUSTER_FLAG_MAX,
};

#ifdef CONFIG_SWAP
#include <linux/swapops.h> /* for swp_offset */
#include <linux/blk_types.h> /* for bio_end_io_t */

static inline unsigned int swp_cluster_offset(swp_entry_t entry)
{
        return swp_offset(entry) % SWAPFILE_CLUSTER;
}

/*
 * Callers of all helpers below must ensure the entry, type, or offset is
 * valid, and protect the swap device with reference count or locks.
 */
static inline struct swap_info_struct *__swap_type_to_info(int type)
{
        struct swap_info_struct *si;

        si = READ_ONCE(swap_info[type]); /* rcu_dereference() */
        VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */
        return si;
}

static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry)
{
        return __swap_type_to_info(swp_type(entry));
}

static inline struct swap_cluster_info *__swap_offset_to_cluster(
                struct swap_info_struct *si, pgoff_t offset)
{
        VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */
        VM_WARN_ON_ONCE(offset >= roundup(si->max, SWAPFILE_CLUSTER));
        return &si->cluster_info[offset / SWAPFILE_CLUSTER];
}

static inline struct swap_cluster_info *__swap_entry_to_cluster(swp_entry_t entry)
{
        return __swap_offset_to_cluster(__swap_entry_to_info(entry),
                                        swp_offset(entry));
}

static __always_inline struct swap_cluster_info *__swap_cluster_lock(
                struct swap_info_struct *si, unsigned long offset, bool irq)
{
        struct swap_cluster_info *ci = __swap_offset_to_cluster(si, offset);

        /*
         * Nothing modifies swap cache in an IRQ context. All access to
         * swap cache is wrapped by swap_cache_* helpers, and swap cache
         * writeback is handled outside of IRQs. Swapin or swapout never
         * occurs in IRQ, and neither does in-place split or replace.
         *
         * Besides, modifying swap cache requires synchronization with
         * swap_map, which was never IRQ safe.
         */
        VM_WARN_ON_ONCE(!in_task());
        VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */
        if (irq)
                spin_lock_irq(&ci->lock);
        else
                spin_lock(&ci->lock);
        return ci;
}

/**
 * swap_cluster_lock - Lock and return the swap cluster of given offset.
 * @si: swap device the cluster belongs to.
 * @offset: the swap entry offset, pointing to a valid slot.
 *
 * Context: The caller must ensure the offset is in the valid range and
 * protect the swap device with reference count or locks.
 */
static inline struct swap_cluster_info *swap_cluster_lock(
                struct swap_info_struct *si, unsigned long offset)
{
        return __swap_cluster_lock(si, offset, false);
}

static inline struct swap_cluster_info *__swap_cluster_get_and_lock(
                const struct folio *folio, bool irq)
{
        VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
        VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
        return __swap_cluster_lock(__swap_entry_to_info(folio->swap),
                                   swp_offset(folio->swap), irq);
}

/*
 * swap_cluster_get_and_lock - Locks the cluster that holds a folio's entries.
 * @folio: The folio.
 *
 * This locks and returns the swap cluster that contains a folio's swap
 * entries. The swap entries of a folio are always in one single cluster.
 * The folio has to be locked so its swap entries won't change and the
 * cluster won't be freed.
 *
 * Context: Caller must ensure the folio is locked and in the swap cache.
 * Return: Pointer to the swap cluster.
 */
static inline struct swap_cluster_info *swap_cluster_get_and_lock(
                const struct folio *folio)
{
        return __swap_cluster_get_and_lock(folio, false);
}

/*
 * swap_cluster_get_and_lock_irq - Locks the cluster that holds a folio's entries.
 * @folio: The folio.
 *
 * Same as swap_cluster_get_and_lock but also disable IRQ.
 *
 * Context: Caller must ensure the folio is locked and in the swap cache.
 * Return: Pointer to the swap cluster.
 */
static inline struct swap_cluster_info *swap_cluster_get_and_lock_irq(
                const struct folio *folio)
{
        return __swap_cluster_get_and_lock(folio, true);
}

static inline void swap_cluster_unlock(struct swap_cluster_info *ci)
{
        spin_unlock(&ci->lock);
}

static inline void swap_cluster_unlock_irq(struct swap_cluster_info *ci)
{
        spin_unlock_irq(&ci->lock);
}

extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp);

/*
 * Below are the core routines for doing swap for a folio.
 * All helpers requires the folio to be locked, and a locked folio
 * in the swap cache pins the swap entries / slots allocated to the
 * folio, swap relies heavily on the swap cache and folio lock for
 * synchronization.
 *
 * folio_alloc_swap(): the entry point for a folio to be swapped
 * out. It allocates swap slots and pins the slots with swap cache.
 * The slots start with a swap count of zero. The slots are pinned
 * by swap cache reference which doesn't contribute to swap count.
 *
 * folio_dup_swap(): increases the swap count of a folio, usually
 * during it gets unmapped and a swap entry is installed to replace
 * it (e.g., swap entry in page table). A swap slot with swap
 * count == 0 can only be increased by this helper.
 *
 * folio_put_swap(): does the opposite thing of folio_dup_swap().
 */
int folio_alloc_swap(struct folio *folio);
int folio_dup_swap(struct folio *folio, struct page *subpage);
void folio_put_swap(struct folio *folio, struct page *subpage);

/* For internal use */
extern void __swap_cluster_free_entries(struct swap_info_struct *si,
                                        struct swap_cluster_info *ci,
                                        unsigned int ci_off, unsigned int nr_pages);

/* linux/mm/page_io.c */
int sio_pool_init(void);
struct swap_iocb;
void swap_read_folio(struct folio *folio, struct swap_iocb **plug);
void __swap_read_unplug(struct swap_iocb *plug);
static inline void swap_read_unplug(struct swap_iocb *plug)
{
        if (unlikely(plug))
                __swap_read_unplug(plug);
}
void swap_write_unplug(struct swap_iocb *sio);
int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug);
void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug);

/* linux/mm/swap_state.c */
extern struct address_space swap_space __read_mostly;
static inline struct address_space *swap_address_space(swp_entry_t entry)
{
        return &swap_space;
}

/*
 * Return the swap device position of the swap entry.
 */
static inline loff_t swap_dev_pos(swp_entry_t entry)
{
        return ((loff_t)swp_offset(entry)) << PAGE_SHIFT;
}

/**
 * folio_matches_swap_entry - Check if a folio matches a given swap entry.
 * @folio: The folio.
 * @entry: The swap entry to check against.
 *
 * Context: The caller should have the folio locked to ensure it's stable
 * and nothing will move it in or out of the swap cache.
 * Return: true or false.
 */
static inline bool folio_matches_swap_entry(const struct folio *folio,
                                            swp_entry_t entry)
{
        swp_entry_t folio_entry = folio->swap;
        long nr_pages = folio_nr_pages(folio);

        VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
        if (!folio_test_swapcache(folio))
                return false;
        VM_WARN_ON_ONCE_FOLIO(!IS_ALIGNED(folio_entry.val, nr_pages), folio);
        return folio_entry.val == round_down(entry.val, nr_pages);
}

/*
 * All swap cache helpers below require the caller to ensure the swap entries
 * used are valid and stabilize the device by any of the following ways:
 * - Hold a reference by get_swap_device(): this ensures a single entry is
 *   valid and increases the swap device's refcount.
 * - Locking a folio in the swap cache: this ensures the folio's swap entries
 *   are valid and pinned, also implies reference to the device.
 * - Locking anything referencing the swap entry: e.g. PTL that protects
 *   swap entries in the page table, similar to locking swap cache folio.
 * - See the comment of get_swap_device() for more complex usage.
 */
bool swap_cache_has_folio(swp_entry_t entry);
struct folio *swap_cache_get_folio(swp_entry_t entry);
void *swap_cache_get_shadow(swp_entry_t entry);
void swap_cache_del_folio(struct folio *folio);
struct folio *swap_cache_alloc_folio(swp_entry_t entry, gfp_t gfp_flags,
                                     struct mempolicy *mpol, pgoff_t ilx,
                                     bool *alloced);
/* Below helpers require the caller to lock and pass in the swap cluster. */
void __swap_cache_add_folio(struct swap_cluster_info *ci,
                            struct folio *folio, swp_entry_t entry);
void __swap_cache_del_folio(struct swap_cluster_info *ci,
                            struct folio *folio, swp_entry_t entry, void *shadow);
void __swap_cache_replace_folio(struct swap_cluster_info *ci,
                                struct folio *old, struct folio *new);

void show_swap_cache_info(void);
void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);
struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                struct vm_area_struct *vma, unsigned long addr,
                struct swap_iocb **plug);
struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
                struct mempolicy *mpol, pgoff_t ilx);
struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,
                struct vm_fault *vmf);
struct folio *swapin_folio(swp_entry_t entry, struct folio *folio);
void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma,
                           unsigned long addr);

static inline unsigned int folio_swap_flags(struct folio *folio)
{
        return __swap_entry_to_info(folio->swap)->flags;
}

/*
 * Return the count of contiguous swap entries that share the same
 * zeromap status as the starting entry. If is_zeromap is not NULL,
 * it will return the zeromap status of the starting entry.
 */
static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
                bool *is_zeromap)
{
        struct swap_info_struct *sis = __swap_entry_to_info(entry);
        unsigned long start = swp_offset(entry);
        unsigned long end = start + max_nr;
        bool first_bit;

        first_bit = test_bit(start, sis->zeromap);
        if (is_zeromap)
                *is_zeromap = first_bit;

        if (max_nr <= 1)
                return max_nr;
        if (first_bit)
                return find_next_zero_bit(sis->zeromap, end, start) - start;
        else
                return find_next_bit(sis->zeromap, end, start) - start;
}

static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
{
        int i;

        /*
         * While allocating a large folio and doing mTHP swapin, we need to
         * ensure all entries are not cached, otherwise, the mTHP folio will
         * be in conflict with the folio in swap cache.
         */
        for (i = 0; i < max_nr; i++) {
                if (swap_cache_has_folio(entry))
                        return i;
                entry.val++;
        }

        return i;
}

#else /* CONFIG_SWAP */
struct swap_iocb;
static inline struct swap_cluster_info *swap_cluster_lock(
        struct swap_info_struct *si, pgoff_t offset, bool irq)
{
        return NULL;
}

static inline struct swap_cluster_info *swap_cluster_get_and_lock(
                struct folio *folio)
{
        return NULL;
}

static inline struct swap_cluster_info *swap_cluster_get_and_lock_irq(
                struct folio *folio)
{
        return NULL;
}

static inline void swap_cluster_unlock(struct swap_cluster_info *ci)
{
}

static inline void swap_cluster_unlock_irq(struct swap_cluster_info *ci)
{
}

static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry)
{
        return NULL;
}

static inline int folio_alloc_swap(struct folio *folio)
{
        return -EINVAL;
}

static inline int folio_dup_swap(struct folio *folio, struct page *page)
{
        return -EINVAL;
}

static inline void folio_put_swap(struct folio *folio, struct page *page)
{
}

static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
{
}

static inline void swap_write_unplug(struct swap_iocb *sio)
{
}

static inline struct address_space *swap_address_space(swp_entry_t entry)
{
        return NULL;
}

static inline bool folio_matches_swap_entry(const struct folio *folio, swp_entry_t entry)
{
        return false;
}

static inline void show_swap_cache_info(void)
{
}

static inline struct folio *swap_cluster_readahead(swp_entry_t entry,
                        gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t ilx)
{
        return NULL;
}

static inline struct folio *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask,
                        struct vm_fault *vmf)
{
        return NULL;
}

static inline struct folio *swapin_folio(swp_entry_t entry, struct folio *folio)
{
        return NULL;
}

static inline void swap_update_readahead(struct folio *folio,
                struct vm_area_struct *vma, unsigned long addr)
{
}

static inline int swap_writeout(struct folio *folio,
                struct swap_iocb **swap_plug)
{
        return 0;
}

static inline int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp)
{
        return -EINVAL;
}

static inline bool swap_cache_has_folio(swp_entry_t entry)
{
        return false;
}

static inline struct folio *swap_cache_get_folio(swp_entry_t entry)
{
        return NULL;
}

static inline void *swap_cache_get_shadow(swp_entry_t entry)
{
        return NULL;
}

static inline void swap_cache_del_folio(struct folio *folio)
{
}

static inline void __swap_cache_del_folio(struct swap_cluster_info *ci,
                struct folio *folio, swp_entry_t entry, void *shadow)
{
}

static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci,
                struct folio *old, struct folio *new)
{
}

static inline unsigned int folio_swap_flags(struct folio *folio)
{
        return 0;
}

static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
                bool *has_zeromap)
{
        return 0;
}

static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
{
        return 0;
}
#endif /* CONFIG_SWAP */
#endif /* _MM_SWAP_H */




















































































































































































































































































































































































































































    5 









   15 














































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    7 






































    7 





























































   16 















    2 

   16 





















































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * IEEE 802.11 defines
 *
 * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
 * <jkmaline@cc.hut.fi>
 * Copyright (c) 2002-2003, Jouni Malinen <jkmaline@cc.hut.fi>
 * Copyright (c) 2005, Devicescape Software, Inc.
 * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
 * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
 * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
 * Copyright (c) 2018 - 2026 Intel Corporation
 */

#ifndef LINUX_IEEE80211_H
#define LINUX_IEEE80211_H

#include <linux/types.h>
#include <linux/if_ether.h>
#include <linux/etherdevice.h>
#include <linux/bitfield.h>
#include <asm/byteorder.h>
#include <linux/unaligned.h>

/*
 * DS bit usage
 *
 * TA = transmitter address
 * RA = receiver address
 * DA = destination address
 * SA = source address
 *
 * ToDS    FromDS  A1(RA)  A2(TA)  A3      A4      Use
 * -----------------------------------------------------------------
 *  0       0       DA      SA      BSSID   -       IBSS/DLS
 *  0       1       DA      BSSID   SA      -       AP -> STA
 *  1       0       BSSID   SA      DA      -       AP <- STA
 *  1       1       RA      TA      DA      SA      unspecified (WDS)
 */

#define FCS_LEN 4

#define IEEE80211_FCTL_VERS                0x0003
#define IEEE80211_FCTL_FTYPE                0x000c
#define IEEE80211_FCTL_STYPE                0x00f0
#define IEEE80211_FCTL_TYPE                (IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)
#define IEEE80211_FCTL_TODS                0x0100
#define IEEE80211_FCTL_FROMDS                0x0200
#define IEEE80211_FCTL_MOREFRAGS        0x0400
#define IEEE80211_FCTL_RETRY                0x0800
#define IEEE80211_FCTL_PM                0x1000
#define IEEE80211_FCTL_MOREDATA                0x2000
#define IEEE80211_FCTL_PROTECTED        0x4000
#define IEEE80211_FCTL_ORDER                0x8000
#define IEEE80211_FCTL_CTL_EXT                0x0f00

#define IEEE80211_SCTL_FRAG                0x000F
#define IEEE80211_SCTL_SEQ                0xFFF0

#define IEEE80211_FTYPE_MGMT                0x0000
#define IEEE80211_FTYPE_CTL                0x0004
#define IEEE80211_FTYPE_DATA                0x0008
#define IEEE80211_FTYPE_EXT                0x000c

/* management */
#define IEEE80211_STYPE_ASSOC_REQ        0x0000
#define IEEE80211_STYPE_ASSOC_RESP        0x0010
#define IEEE80211_STYPE_REASSOC_REQ        0x0020
#define IEEE80211_STYPE_REASSOC_RESP        0x0030
#define IEEE80211_STYPE_PROBE_REQ        0x0040
#define IEEE80211_STYPE_PROBE_RESP        0x0050
#define IEEE80211_STYPE_BEACON                0x0080
#define IEEE80211_STYPE_ATIM                0x0090
#define IEEE80211_STYPE_DISASSOC        0x00A0
#define IEEE80211_STYPE_AUTH                0x00B0
#define IEEE80211_STYPE_DEAUTH                0x00C0
#define IEEE80211_STYPE_ACTION                0x00D0

/* control */
#define IEEE80211_STYPE_TRIGGER                0x0020
#define IEEE80211_STYPE_CTL_EXT                0x0060
#define IEEE80211_STYPE_BACK_REQ        0x0080
#define IEEE80211_STYPE_BACK                0x0090
#define IEEE80211_STYPE_PSPOLL                0x00A0
#define IEEE80211_STYPE_RTS                0x00B0
#define IEEE80211_STYPE_CTS                0x00C0
#define IEEE80211_STYPE_ACK                0x00D0
#define IEEE80211_STYPE_CFEND                0x00E0
#define IEEE80211_STYPE_CFENDACK        0x00F0

/* data */
#define IEEE80211_STYPE_DATA                        0x0000
#define IEEE80211_STYPE_DATA_CFACK                0x0010
#define IEEE80211_STYPE_DATA_CFPOLL                0x0020
#define IEEE80211_STYPE_DATA_CFACKPOLL                0x0030
#define IEEE80211_STYPE_NULLFUNC                0x0040
#define IEEE80211_STYPE_CFACK                        0x0050
#define IEEE80211_STYPE_CFPOLL                        0x0060
#define IEEE80211_STYPE_CFACKPOLL                0x0070
#define IEEE80211_STYPE_QOS_DATA                0x0080
#define IEEE80211_STYPE_QOS_DATA_CFACK                0x0090
#define IEEE80211_STYPE_QOS_DATA_CFPOLL                0x00A0
#define IEEE80211_STYPE_QOS_DATA_CFACKPOLL        0x00B0
#define IEEE80211_STYPE_QOS_NULLFUNC                0x00C0
#define IEEE80211_STYPE_QOS_CFACK                0x00D0
#define IEEE80211_STYPE_QOS_CFPOLL                0x00E0
#define IEEE80211_STYPE_QOS_CFACKPOLL                0x00F0

/* extension, added by 802.11ad */
#define IEEE80211_STYPE_DMG_BEACON                0x0000
#define IEEE80211_STYPE_S1G_BEACON                0x0010

#define IEEE80211_NDP_FTYPE_CTS                        0
#define IEEE80211_NDP_FTYPE_CF_END                0
#define IEEE80211_NDP_FTYPE_PS_POLL                1
#define IEEE80211_NDP_FTYPE_ACK                        2
#define IEEE80211_NDP_FTYPE_PS_POLL_ACK                3
#define IEEE80211_NDP_FTYPE_BA                        4
#define IEEE80211_NDP_FTYPE_BF_REPORT_POLL        5
#define IEEE80211_NDP_FTYPE_PAGING                6
#define IEEE80211_NDP_FTYPE_PREQ                7

#define SM64(f, v)        ((((u64)v) << f##_S) & f)

/* NDP CMAC frame fields */
#define IEEE80211_NDP_FTYPE                    0x0000000000000007
#define IEEE80211_NDP_FTYPE_S                  0x0000000000000000

/* 1M Probe Request 11ah 9.9.3.1.1 */
#define IEEE80211_NDP_1M_PREQ_ANO      0x0000000000000008
#define IEEE80211_NDP_1M_PREQ_ANO_S                     3
#define IEEE80211_NDP_1M_PREQ_CSSID    0x00000000000FFFF0
#define IEEE80211_NDP_1M_PREQ_CSSID_S                   4
#define IEEE80211_NDP_1M_PREQ_RTYPE    0x0000000000100000
#define IEEE80211_NDP_1M_PREQ_RTYPE_S                  20
#define IEEE80211_NDP_1M_PREQ_RSV      0x0000000001E00000
#define IEEE80211_NDP_1M_PREQ_RSV      0x0000000001E00000
/* 2M Probe Request 11ah 9.9.3.1.2 */
#define IEEE80211_NDP_2M_PREQ_ANO      0x0000000000000008
#define IEEE80211_NDP_2M_PREQ_ANO_S                     3
#define IEEE80211_NDP_2M_PREQ_CSSID    0x0000000FFFFFFFF0
#define IEEE80211_NDP_2M_PREQ_CSSID_S                   4
#define IEEE80211_NDP_2M_PREQ_RTYPE    0x0000001000000000
#define IEEE80211_NDP_2M_PREQ_RTYPE_S                  36

#define IEEE80211_ANO_NETTYPE_WILD              15

/* control extension - for IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CTL_EXT */
#define IEEE80211_CTL_EXT_POLL                0x2000
#define IEEE80211_CTL_EXT_SPR                0x3000
#define IEEE80211_CTL_EXT_GRANT        0x4000
#define IEEE80211_CTL_EXT_DMG_CTS        0x5000
#define IEEE80211_CTL_EXT_DMG_DTS        0x6000
#define IEEE80211_CTL_EXT_SSW                0x8000
#define IEEE80211_CTL_EXT_SSW_FBACK        0x9000
#define IEEE80211_CTL_EXT_SSW_ACK        0xa000


#define IEEE80211_SN_MASK                ((IEEE80211_SCTL_SEQ) >> 4)
#define IEEE80211_MAX_SN                IEEE80211_SN_MASK
#define IEEE80211_SN_MODULO                (IEEE80211_MAX_SN + 1)


/* PV1 Layout IEEE 802.11-2020 9.8.3.1 */
#define IEEE80211_PV1_FCTL_VERS                0x0003
#define IEEE80211_PV1_FCTL_FTYPE        0x001c
#define IEEE80211_PV1_FCTL_STYPE        0x00e0
#define IEEE80211_PV1_FCTL_FROMDS                0x0100
#define IEEE80211_PV1_FCTL_MOREFRAGS        0x0200
#define IEEE80211_PV1_FCTL_PM                0x0400
#define IEEE80211_PV1_FCTL_MOREDATA        0x0800
#define IEEE80211_PV1_FCTL_PROTECTED        0x1000
#define IEEE80211_PV1_FCTL_END_SP       0x2000
#define IEEE80211_PV1_FCTL_RELAYED      0x4000
#define IEEE80211_PV1_FCTL_ACK_POLICY   0x8000
#define IEEE80211_PV1_FCTL_CTL_EXT        0x0f00

static inline bool ieee80211_sn_less(u16 sn1, u16 sn2)
{
        return ((sn1 - sn2) & IEEE80211_SN_MASK) > (IEEE80211_SN_MODULO >> 1);
}

static inline bool ieee80211_sn_less_eq(u16 sn1, u16 sn2)
{
        return ((sn2 - sn1) & IEEE80211_SN_MASK) <= (IEEE80211_SN_MODULO >> 1);
}

static inline u16 ieee80211_sn_add(u16 sn1, u16 sn2)
{
        return (sn1 + sn2) & IEEE80211_SN_MASK;
}

static inline u16 ieee80211_sn_inc(u16 sn)
{
        return ieee80211_sn_add(sn, 1);
}

static inline u16 ieee80211_sn_sub(u16 sn1, u16 sn2)
{
        return (sn1 - sn2) & IEEE80211_SN_MASK;
}

#define IEEE80211_SEQ_TO_SN(seq)        (((seq) & IEEE80211_SCTL_SEQ) >> 4)
#define IEEE80211_SN_TO_SEQ(ssn)        (((ssn) << 4) & IEEE80211_SCTL_SEQ)

/* miscellaneous IEEE 802.11 constants */
#define IEEE80211_MAX_FRAG_THRESHOLD        2352
#define IEEE80211_MAX_RTS_THRESHOLD        2353
#define IEEE80211_MAX_AID                2007
#define IEEE80211_MAX_AID_S1G                8191
#define IEEE80211_MAX_TIM_LEN                251
#define IEEE80211_MAX_MESH_PEERINGS        63

/* Maximum size for the MA-UNITDATA primitive, 802.11 standard section
   6.2.1.1.2.

   802.11e clarifies the figure in section 7.1.2. The frame body is
   up to 2304 octets long (maximum MSDU size) plus any crypt overhead. */
#define IEEE80211_MAX_DATA_LEN                2304
/* 802.11ad extends maximum MSDU size for DMG (freq > 40Ghz) networks
 * to 7920 bytes, see 8.2.3 General frame format
 */
#define IEEE80211_MAX_DATA_LEN_DMG        7920
/* 30 byte 4 addr hdr, 2 byte QoS, 2304 byte MSDU, 12 byte crypt, 4 byte FCS */
#define IEEE80211_MAX_FRAME_LEN                2352

#define IEEE80211_MAX_SSID_LEN                32

#define IEEE80211_FIRST_TSPEC_TSID        8
#define IEEE80211_NUM_TIDS                16

/* number of user priorities 802.11 uses */
#define IEEE80211_NUM_UPS                8
/* number of ACs */
#define IEEE80211_NUM_ACS                4

#define IEEE80211_QOS_CTL_LEN                2
/* 1d tag mask */
#define IEEE80211_QOS_CTL_TAG1D_MASK                0x0007
/* TID mask */
#define IEEE80211_QOS_CTL_TID_MASK                0x000f
/* EOSP */
#define IEEE80211_QOS_CTL_EOSP                        0x0010
/* ACK policy */
#define IEEE80211_QOS_CTL_ACK_POLICY_NORMAL        0x0000
#define IEEE80211_QOS_CTL_ACK_POLICY_NOACK        0x0020
#define IEEE80211_QOS_CTL_ACK_POLICY_NO_EXPL        0x0040
#define IEEE80211_QOS_CTL_ACK_POLICY_BLOCKACK        0x0060
#define IEEE80211_QOS_CTL_ACK_POLICY_MASK        0x0060
/* A-MSDU 802.11n */
#define IEEE80211_QOS_CTL_A_MSDU_PRESENT        0x0080
/* Mesh Control 802.11s */
#define IEEE80211_QOS_CTL_MESH_CONTROL_PRESENT  0x0100

/* Mesh Power Save Level */
#define IEEE80211_QOS_CTL_MESH_PS_LEVEL                0x0200
/* Mesh Receiver Service Period Initiated */
#define IEEE80211_QOS_CTL_RSPI                        0x0400

/* U-APSD queue for WMM IEs sent by AP */
#define IEEE80211_WMM_IE_AP_QOSINFO_UAPSD        (1<<7)
#define IEEE80211_WMM_IE_AP_QOSINFO_PARAM_SET_CNT_MASK        0x0f

/* U-APSD queues for WMM IEs sent by STA */
#define IEEE80211_WMM_IE_STA_QOSINFO_AC_VO        (1<<0)
#define IEEE80211_WMM_IE_STA_QOSINFO_AC_VI        (1<<1)
#define IEEE80211_WMM_IE_STA_QOSINFO_AC_BK        (1<<2)
#define IEEE80211_WMM_IE_STA_QOSINFO_AC_BE        (1<<3)
#define IEEE80211_WMM_IE_STA_QOSINFO_AC_MASK        0x0f

/* U-APSD max SP length for WMM IEs sent by STA */
#define IEEE80211_WMM_IE_STA_QOSINFO_SP_ALL        0x00
#define IEEE80211_WMM_IE_STA_QOSINFO_SP_2        0x01
#define IEEE80211_WMM_IE_STA_QOSINFO_SP_4        0x02
#define IEEE80211_WMM_IE_STA_QOSINFO_SP_6        0x03
#define IEEE80211_WMM_IE_STA_QOSINFO_SP_MASK        0x03
#define IEEE80211_WMM_IE_STA_QOSINFO_SP_SHIFT        5

/* trigger type within common_info of trigger frame */
#define IEEE80211_TRIGGER_TYPE_MASK                0xf
#define IEEE80211_TRIGGER_TYPE_BASIC                0x0
#define IEEE80211_TRIGGER_TYPE_BFRP                0x1
#define IEEE80211_TRIGGER_TYPE_MU_BAR                0x2
#define IEEE80211_TRIGGER_TYPE_MU_RTS                0x3
#define IEEE80211_TRIGGER_TYPE_BSRP                0x4
#define IEEE80211_TRIGGER_TYPE_GCR_MU_BAR        0x5
#define IEEE80211_TRIGGER_TYPE_BQRP                0x6
#define IEEE80211_TRIGGER_TYPE_NFRP                0x7

/* UL-bandwidth within common_info of trigger frame */
#define IEEE80211_TRIGGER_ULBW_MASK                0xc0000
#define IEEE80211_TRIGGER_ULBW_20MHZ                0x0
#define IEEE80211_TRIGGER_ULBW_40MHZ                0x1
#define IEEE80211_TRIGGER_ULBW_80MHZ                0x2
#define IEEE80211_TRIGGER_ULBW_160_80P80MHZ        0x3

struct ieee80211_hdr {
        __le16 frame_control;
        __le16 duration_id;
        struct_group(addrs,
                u8 addr1[ETH_ALEN];
                u8 addr2[ETH_ALEN];
                u8 addr3[ETH_ALEN];
        );
        __le16 seq_ctrl;
        u8 addr4[ETH_ALEN];
} __packed __aligned(2);

struct ieee80211_hdr_3addr {
        __le16 frame_control;
        __le16 duration_id;
        u8 addr1[ETH_ALEN];
        u8 addr2[ETH_ALEN];
        u8 addr3[ETH_ALEN];
        __le16 seq_ctrl;
} __packed __aligned(2);

struct ieee80211_qos_hdr {
        __le16 frame_control;
        __le16 duration_id;
        u8 addr1[ETH_ALEN];
        u8 addr2[ETH_ALEN];
        u8 addr3[ETH_ALEN];
        __le16 seq_ctrl;
        __le16 qos_ctrl;
} __packed __aligned(2);

struct ieee80211_qos_hdr_4addr {
        __le16 frame_control;
        __le16 duration_id;
        u8 addr1[ETH_ALEN];
        u8 addr2[ETH_ALEN];
        u8 addr3[ETH_ALEN];
        __le16 seq_ctrl;
        u8 addr4[ETH_ALEN];
        __le16 qos_ctrl;
} __packed __aligned(2);

struct ieee80211_trigger {
        __le16 frame_control;
        __le16 duration;
        u8 ra[ETH_ALEN];
        u8 ta[ETH_ALEN];
        __le64 common_info;
        u8 variable[];
} __packed __aligned(2);

/**
 * ieee80211_has_tods - check if IEEE80211_FCTL_TODS is set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame has to-DS set
 */
static inline bool ieee80211_has_tods(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_TODS)) != 0;
}

/**
 * ieee80211_has_fromds - check if IEEE80211_FCTL_FROMDS is set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame has from-DS set
 */
static inline bool ieee80211_has_fromds(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FROMDS)) != 0;
}

/**
 * ieee80211_has_a4 - check if IEEE80211_FCTL_TODS and IEEE80211_FCTL_FROMDS are set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not it's a 4-address frame (from-DS and to-DS set)
 */
static inline bool ieee80211_has_a4(__le16 fc)
{
        __le16 tmp = cpu_to_le16(IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS);
        return (fc & tmp) == tmp;
}

/**
 * ieee80211_has_morefrags - check if IEEE80211_FCTL_MOREFRAGS is set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame has more fragments (more frags bit set)
 */
static inline bool ieee80211_has_morefrags(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_MOREFRAGS)) != 0;
}

/**
 * ieee80211_has_retry - check if IEEE80211_FCTL_RETRY is set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the retry flag is set
 */
static inline bool ieee80211_has_retry(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_RETRY)) != 0;
}

/**
 * ieee80211_has_pm - check if IEEE80211_FCTL_PM is set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the power management flag is set
 */
static inline bool ieee80211_has_pm(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_PM)) != 0;
}

/**
 * ieee80211_has_moredata - check if IEEE80211_FCTL_MOREDATA is set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the more data flag is set
 */
static inline bool ieee80211_has_moredata(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_MOREDATA)) != 0;
}

/**
 * ieee80211_has_protected - check if IEEE80211_FCTL_PROTECTED is set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the protected flag is set
 */
static inline bool ieee80211_has_protected(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_PROTECTED)) != 0;
}

/**
 * ieee80211_has_order - check if IEEE80211_FCTL_ORDER is set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the order flag is set
 */
static inline bool ieee80211_has_order(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_ORDER)) != 0;
}

/**
 * ieee80211_is_mgmt - check if type is IEEE80211_FTYPE_MGMT
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame type is management
 */
static inline bool ieee80211_is_mgmt(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT);
}

/**
 * ieee80211_is_ctl - check if type is IEEE80211_FTYPE_CTL
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame type is control
 */
static inline bool ieee80211_is_ctl(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL);
}

/**
 * ieee80211_is_data - check if type is IEEE80211_FTYPE_DATA
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a data frame
 */
static inline bool ieee80211_is_data(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_DATA);
}

/**
 * ieee80211_is_ext - check if type is IEEE80211_FTYPE_EXT
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame type is extended
 */
static inline bool ieee80211_is_ext(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_EXT);
}


/**
 * ieee80211_is_data_qos - check if type is IEEE80211_FTYPE_DATA and IEEE80211_STYPE_QOS_DATA is set
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a QoS data frame
 */
static inline bool ieee80211_is_data_qos(__le16 fc)
{
        /*
         * mask with QOS_DATA rather than IEEE80211_FCTL_STYPE as we just need
         * to check the one bit
         */
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_STYPE_QOS_DATA)) ==
               cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_DATA);
}

/**
 * ieee80211_is_data_present - check if type is IEEE80211_FTYPE_DATA and has data
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a QoS data frame that has data
 *        (i.e. is not null data)
 */
static inline bool ieee80211_is_data_present(__le16 fc)
{
        /*
         * mask with 0x40 and test that that bit is clear to only return true
         * for the data-containing substypes.
         */
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | 0x40)) ==
               cpu_to_le16(IEEE80211_FTYPE_DATA);
}

/**
 * ieee80211_is_assoc_req - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_ASSOC_REQ
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is an association request
 */
static inline bool ieee80211_is_assoc_req(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_ASSOC_REQ);
}

/**
 * ieee80211_is_assoc_resp - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_ASSOC_RESP
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is an association response
 */
static inline bool ieee80211_is_assoc_resp(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_ASSOC_RESP);
}

/**
 * ieee80211_is_reassoc_req - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_REASSOC_REQ
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a reassociation request
 */
static inline bool ieee80211_is_reassoc_req(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_REASSOC_REQ);
}

/**
 * ieee80211_is_reassoc_resp - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_REASSOC_RESP
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a reassociation response
 */
static inline bool ieee80211_is_reassoc_resp(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_REASSOC_RESP);
}

/**
 * ieee80211_is_probe_req - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_PROBE_REQ
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a probe request
 */
static inline bool ieee80211_is_probe_req(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_PROBE_REQ);
}

/**
 * ieee80211_is_probe_resp - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_PROBE_RESP
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a probe response
 */
static inline bool ieee80211_is_probe_resp(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_PROBE_RESP);
}

/**
 * ieee80211_is_beacon - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_BEACON
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a (regular, not S1G) beacon
 */
static inline bool ieee80211_is_beacon(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_BEACON);
}

/**
 * ieee80211_is_atim - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_ATIM
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is an ATIM frame
 */
static inline bool ieee80211_is_atim(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_ATIM);
}

/**
 * ieee80211_is_disassoc - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_DISASSOC
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a disassociation frame
 */
static inline bool ieee80211_is_disassoc(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_DISASSOC);
}

/**
 * ieee80211_is_auth - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_AUTH
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is an authentication frame
 */
static inline bool ieee80211_is_auth(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_AUTH);
}

/**
 * ieee80211_is_deauth - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_DEAUTH
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a deauthentication frame
 */
static inline bool ieee80211_is_deauth(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_DEAUTH);
}

/**
 * ieee80211_is_action - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_ACTION
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is an action frame
 */
static inline bool ieee80211_is_action(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_ACTION);
}

/**
 * ieee80211_is_back_req - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_BACK_REQ
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a block-ACK request frame
 */
static inline bool ieee80211_is_back_req(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_BACK_REQ);
}

/**
 * ieee80211_is_back - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_BACK
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a block-ACK frame
 */
static inline bool ieee80211_is_back(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_BACK);
}

/**
 * ieee80211_is_pspoll - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_PSPOLL
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a PS-poll frame
 */
static inline bool ieee80211_is_pspoll(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_PSPOLL);
}

/**
 * ieee80211_is_rts - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_RTS
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is an RTS frame
 */
static inline bool ieee80211_is_rts(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_RTS);
}

/**
 * ieee80211_is_cts - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_CTS
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a CTS frame
 */
static inline bool ieee80211_is_cts(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CTS);
}

/**
 * ieee80211_is_ack - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_ACK
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is an ACK frame
 */
static inline bool ieee80211_is_ack(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_ACK);
}

/**
 * ieee80211_is_cfend - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_CFEND
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a CF-end frame
 */
static inline bool ieee80211_is_cfend(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CFEND);
}

/**
 * ieee80211_is_cfendack - check if IEEE80211_FTYPE_CTL && IEEE80211_STYPE_CFENDACK
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a CF-end-ack frame
 */
static inline bool ieee80211_is_cfendack(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_CFENDACK);
}

/**
 * ieee80211_is_nullfunc - check if frame is a regular (non-QoS) nullfunc frame
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a nullfunc frame
 */
static inline bool ieee80211_is_nullfunc(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_NULLFUNC);
}

/**
 * ieee80211_is_qos_nullfunc - check if frame is a QoS nullfunc frame
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a QoS nullfunc frame
 */
static inline bool ieee80211_is_qos_nullfunc(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_NULLFUNC);
}

/**
 * ieee80211_is_trigger - check if frame is trigger frame
 * @fc: frame control field in little-endian byteorder
 * Return: whether or not the frame is a trigger frame
 */
static inline bool ieee80211_is_trigger(__le16 fc)
{
        return (fc & cpu_to_le16(IEEE80211_FCTL_FTYPE | IEEE80211_FCTL_STYPE)) ==
               cpu_to_le16(IEEE80211_FTYPE_CTL | IEEE80211_STYPE_TRIGGER);
}

/**
 * ieee80211_is_any_nullfunc - check if frame is regular or QoS nullfunc frame
 * @fc: frame control bytes in little-endian byteorder
 * Return: whether or not the frame is a nullfunc or QoS nullfunc frame
 */
static inline bool ieee80211_is_any_nullfunc(__le16 fc)
{
        return (ieee80211_is_nullfunc(fc) || ieee80211_is_qos_nullfunc(fc));
}

/**
 * ieee80211_is_first_frag - check if IEEE80211_SCTL_FRAG is not set
 * @seq_ctrl: frame sequence control bytes in little-endian byteorder
 * Return: whether or not the frame is the first fragment (also true if
 *        it's not fragmented at all)
 */
static inline bool ieee80211_is_first_frag(__le16 seq_ctrl)
{
        return (seq_ctrl & cpu_to_le16(IEEE80211_SCTL_FRAG)) == 0;
}

/**
 * ieee80211_is_frag - check if a frame is a fragment
 * @hdr: 802.11 header of the frame
 * Return: whether or not the frame is a fragment
 */
static inline bool ieee80211_is_frag(struct ieee80211_hdr *hdr)
{
        return ieee80211_has_morefrags(hdr->frame_control) ||
               hdr->seq_ctrl & cpu_to_le16(IEEE80211_SCTL_FRAG);
}

static inline u16 ieee80211_get_sn(struct ieee80211_hdr *hdr)
{
        return le16_get_bits(hdr->seq_ctrl, IEEE80211_SCTL_SEQ);
}

/**
 * struct ieee80211_quiet_ie - Quiet element
 * @count: Quiet Count
 * @period: Quiet Period
 * @duration: Quiet Duration
 * @offset: Quiet Offset
 *
 * This structure represents the payload of the "Quiet element" as
 * described in IEEE Std 802.11-2020 section 9.4.2.22.
 */
struct ieee80211_quiet_ie {
        u8 count;
        u8 period;
        __le16 duration;
        __le16 offset;
} __packed;

/**
 * struct ieee80211_msrment_ie - Measurement element
 * @token: Measurement Token
 * @mode: Measurement Report Mode
 * @type: Measurement Type
 * @request: Measurement Request or Measurement Report
 *
 * This structure represents the payload of both the "Measurement
 * Request element" and the "Measurement Report element" as described
 * in IEEE Std 802.11-2020 sections 9.4.2.20 and 9.4.2.21.
 */
struct ieee80211_msrment_ie {
        u8 token;
        u8 mode;
        u8 type;
        u8 request[];
} __packed;

/**
 * struct ieee80211_channel_sw_ie - Channel Switch Announcement element
 * @mode: Channel Switch Mode
 * @new_ch_num: New Channel Number
 * @count: Channel Switch Count
 *
 * This structure represents the payload of the "Channel Switch
 * Announcement element" as described in IEEE Std 802.11-2020 section
 * 9.4.2.18.
 */
struct ieee80211_channel_sw_ie {
        u8 mode;
        u8 new_ch_num;
        u8 count;
} __packed;

/**
 * struct ieee80211_ext_chansw_ie - Extended Channel Switch Announcement element
 * @mode: Channel Switch Mode
 * @new_operating_class: New Operating Class
 * @new_ch_num: New Channel Number
 * @count: Channel Switch Count
 *
 * This structure represents the "Extended Channel Switch Announcement
 * element" as described in IEEE Std 802.11-2020 section 9.4.2.52.
 */
struct ieee80211_ext_chansw_ie {
        u8 mode;
        u8 new_operating_class;
        u8 new_ch_num;
        u8 count;
} __packed;

/**
 * struct ieee80211_sec_chan_offs_ie - secondary channel offset IE
 * @sec_chan_offs: secondary channel offset, uses IEEE80211_HT_PARAM_CHA_SEC_*
 *        values here
 * This structure represents the "Secondary Channel Offset element"
 */
struct ieee80211_sec_chan_offs_ie {
        u8 sec_chan_offs;
} __packed;

/**
 * struct ieee80211_wide_bw_chansw_ie - wide bandwidth channel switch IE
 * @new_channel_width: New Channel Width
 * @new_center_freq_seg0: New Channel Center Frequency Segment 0
 * @new_center_freq_seg1: New Channel Center Frequency Segment 1
 *
 * This structure represents the payload of the "Wide Bandwidth
 * Channel Switch element" as described in IEEE Std 802.11-2020
 * section 9.4.2.160.
 */
struct ieee80211_wide_bw_chansw_ie {
        u8 new_channel_width;
        u8 new_center_freq_seg0, new_center_freq_seg1;
} __packed;

/**
 * struct ieee80211_tim_ie - Traffic Indication Map information element
 * @dtim_count: DTIM Count
 * @dtim_period: DTIM Period
 * @bitmap_ctrl: Bitmap Control
 * @required_octet: "Syntatic sugar" to force the struct size to the
 *                  minimum valid size when carried in a non-S1G PPDU
 * @virtual_map: Partial Virtual Bitmap
 *
 * This structure represents the payload of the "TIM element" as
 * described in IEEE Std 802.11-2020 section 9.4.2.5. Note that this
 * definition is only applicable when the element is carried in a
 * non-S1G PPDU. When the TIM is carried in an S1G PPDU, the Bitmap
 * Control and Partial Virtual Bitmap may not be present.
 */
struct ieee80211_tim_ie {
        u8 dtim_count;
        u8 dtim_period;
        u8 bitmap_ctrl;
        union {
                u8 required_octet;
                DECLARE_FLEX_ARRAY(u8, virtual_map);
        };
} __packed;

#define WLAN_SA_QUERY_TR_ID_LEN 2
#define WLAN_MEMBERSHIP_LEN 8
#define WLAN_USER_POSITION_LEN 16

/**
 * struct ieee80211_tpc_report_ie - TPC Report element
 * @tx_power: Transmit Power
 * @link_margin: Link Margin
 *
 * This structure represents the payload of the "TPC Report element" as
 * described in IEEE Std 802.11-2020 section 9.4.2.16.
 */
struct ieee80211_tpc_report_ie {
        u8 tx_power;
        u8 link_margin;
} __packed;

#define IEEE80211_ADDBA_EXT_FRAG_LEVEL_MASK        GENMASK(2, 1)
#define IEEE80211_ADDBA_EXT_FRAG_LEVEL_SHIFT        1
#define IEEE80211_ADDBA_EXT_NO_FRAG                BIT(0)
#define IEEE80211_ADDBA_EXT_BUF_SIZE_MASK        GENMASK(7, 5)
#define IEEE80211_ADDBA_EXT_BUF_SIZE_SHIFT        10

struct ieee80211_addba_ext_ie {
        u8 data;
} __packed;

struct ieee80211_ext {
        __le16 frame_control;
        __le16 duration;
        union {
                struct {
                        u8 sa[ETH_ALEN];
                        __le32 timestamp;
                        u8 change_seq;
                        u8 variable[];
                } __packed s1g_beacon;
        } u;
} __packed __aligned(2);

/**
 * struct ieee80211_bss_load_elem - BSS Load elemen
 *
 * Defined in section 9.4.2.26 in IEEE 802.11-REVme D4.1
 *
 * @sta_count: total number of STAs currently associated with the AP.
 * @channel_util: Percentage of time that the access point sensed the channel
 *        was busy. This value is in range [0, 255], the highest value means
 *        100% busy.
 * @avail_admission_capa: remaining amount of medium time used for admission
 *        control.
 */
struct ieee80211_bss_load_elem {
        __le16 sta_count;
        u8 channel_util;
        __le16 avail_admission_capa;
} __packed;

struct ieee80211_mgmt {
        __le16 frame_control;
        __le16 duration;
        u8 da[ETH_ALEN];
        u8 sa[ETH_ALEN];
        u8 bssid[ETH_ALEN];
        __le16 seq_ctrl;
        union {
                struct {
                        __le16 auth_alg;
                        __le16 auth_transaction;
                        __le16 status_code;
                        /* possibly followed by Challenge text */
                        u8 variable[];
                } __packed auth;
                struct {
                        __le16 reason_code;
                } __packed deauth;
                struct {
                        __le16 capab_info;
                        __le16 listen_interval;
                        /* followed by SSID and Supported rates */
                        u8 variable[];
                } __packed assoc_req;
                struct {
                        __le16 capab_info;
                        __le16 status_code;
                        __le16 aid;
                        /* followed by Supported rates */
                        u8 variable[];
                } __packed assoc_resp, reassoc_resp;
                struct {
                        __le16 capab_info;
                        __le16 status_code;
                        u8 variable[];
                } __packed s1g_assoc_resp, s1g_reassoc_resp;
                struct {
                        __le16 capab_info;
                        __le16 listen_interval;
                        u8 current_ap[ETH_ALEN];
                        /* followed by SSID and Supported rates */
                        u8 variable[];
                } __packed reassoc_req;
                struct {
                        __le16 reason_code;
                } __packed disassoc;
                struct {
                        __le64 timestamp;
                        __le16 beacon_int;
                        __le16 capab_info;
                        /* followed by some of SSID, Supported rates,
                         * FH Params, DS Params, CF Params, IBSS Params, TIM */
                        u8 variable[];
                } __packed beacon;
                struct {
                        /* only variable items: SSID, Supported rates */
                        DECLARE_FLEX_ARRAY(u8, variable);
                } __packed probe_req;
                struct {
                        __le64 timestamp;
                        __le16 beacon_int;
                        __le16 capab_info;
                        /* followed by some of SSID, Supported rates,
                         * FH Params, DS Params, CF Params, IBSS Params */
                        u8 variable[];
                } __packed probe_resp;
                struct {
                        u8 category;
                        u8 action_code;
                        union {
                                struct {
                                        u8 dialog_token;
                                        u8 status_code;
                                        u8 variable[];
                                } __packed wme_action;
                                struct{
                                        u8 no_fixed_fields[0];
                                        u8 variable[];
                                } __packed chan_switch;
                                struct{
                                        struct ieee80211_ext_chansw_ie data;
                                        u8 variable[];
                                } __packed ext_chan_switch;
                                struct{
                                        u8 dialog_token;
                                        u8 element_id;
                                        u8 length;
                                        struct ieee80211_msrment_ie msr_elem;
                                } __packed measurement;
                                struct{
                                        u8 dialog_token;
                                        __le16 capab;
                                        __le16 timeout;
                                        __le16 start_seq_num;
                                        /* followed by BA Extension */
                                        u8 variable[];
                                } __packed addba_req;
                                struct{
                                        u8 dialog_token;
                                        __le16 status;
                                        __le16 capab;
                                        __le16 timeout;
                                        /* followed by BA Extension */
                                        u8 variable[];
                                } __packed addba_resp;
                                struct{
                                        __le16 params;
                                        __le16 reason_code;
                                } __packed delba;
                                struct {
                                        u8 no_fixed_fields[0];
                                        u8 variable[];
                                } __packed self_prot;
                                struct{
                                        u8 no_fixed_fields[0];
                                        u8 variable[];
                                } __packed mesh_action;
                                struct {
                                        u8 trans_id[WLAN_SA_QUERY_TR_ID_LEN];
                                } __packed sa_query;
                                struct {
                                        u8 smps_control;
                                } __packed ht_smps;
                                struct {
                                        u8 chanwidth;
                                } __packed ht_notify_cw;
                                struct {
                                        u8 dialog_token;
                                        __le16 capability;
                                        u8 variable[];
                                } __packed tdls_discover_resp;
                                struct {
                                        u8 operating_mode;
                                } __packed vht_opmode_notif;
                                struct {
                                        u8 membership[WLAN_MEMBERSHIP_LEN];
                                        u8 position[WLAN_USER_POSITION_LEN];
                                } __packed vht_group_notif;
                                struct {
                                        u8 dialog_token;
                                        u8 tpc_elem_id;
                                        u8 tpc_elem_length;
                                        struct ieee80211_tpc_report_ie tpc;
                                } __packed tpc_report;
                                struct {
                                        u8 dialog_token;
                                        u8 follow_up;
                                        u8 tod[6];
                                        u8 toa[6];
                                        __le16 tod_error;
                                        __le16 toa_error;
                                        u8 variable[];
                                } __packed ftm;
                                struct {
                                        u8 no_fixed_fields[0];
                                        u8 variable[];
                                } __packed s1g;
                                struct {
                                        u8 dialog_token;
                                        u8 follow_up;
                                        u32 tod;
                                        u32 toa;
                                        u8 max_tod_error;
                                        u8 max_toa_error;
                                } __packed wnm_timing_msr;
                                struct {
                                        u8 dialog_token;
                                        u8 variable[];
                                } __packed ttlm_req;
                                struct {
                                        u8 dialog_token;
                                        __le16 status_code;
                                        u8 variable[];
                                } __packed ttlm_res;
                                struct {
                                        u8 no_fixed_fields[0];
                                        /* no variable fields either */
                                } __packed ttlm_tear_down;
                                struct {
                                        u8 dialog_token;
                                        u8 variable[];
                                } __packed ml_reconf_req;
                                struct {
                                        u8 dialog_token;
                                        u8 count;
                                        u8 variable[];
                                } __packed ml_reconf_resp;
                                struct {
                                        u8 no_fixed_fields[0];
                                        u8 variable[];
                                } __packed epcs;
                                struct {
                                        u8 dialog_token;
                                        u8 control;
                                        u8 variable[];
                                } __packed eml_omn;
                        };
                } __packed action;
                DECLARE_FLEX_ARRAY(u8, body); /* Generic frame body */
        } u;
} __packed __aligned(2);

/* Supported rates membership selectors */
#define BSS_MEMBERSHIP_SELECTOR_HT_PHY        127
#define BSS_MEMBERSHIP_SELECTOR_VHT_PHY        126
#define BSS_MEMBERSHIP_SELECTOR_GLK        125
#define BSS_MEMBERSHIP_SELECTOR_EPD        124
#define BSS_MEMBERSHIP_SELECTOR_SAE_H2E 123
#define BSS_MEMBERSHIP_SELECTOR_HE_PHY        122
#define BSS_MEMBERSHIP_SELECTOR_EHT_PHY        121
#define BSS_MEMBERSHIP_SELECTOR_UHR_PHY        120

#define BSS_MEMBERSHIP_SELECTOR_MIN        BSS_MEMBERSHIP_SELECTOR_UHR_PHY

#define IEEE80211_MIN_ACTION_SIZE(type)        offsetofend(struct ieee80211_mgmt, u.action.type)

/* Link Reconfiguration Status Duple field */
struct ieee80211_ml_reconf_status {
        u8 info;
        __le16 status;
} __packed;

#define IEEE80211_ML_RECONF_LINK_ID_MASK        0xf

/* Management MIC information element (IEEE 802.11w) for CMAC */
struct ieee80211_mmie {
        u8 element_id;
        u8 length;
        __le16 key_id;
        u8 sequence_number[6];
        u8 mic[8];
} __packed;

/* Management MIC information element (IEEE 802.11w) for GMAC and CMAC-256 */
struct ieee80211_mmie_16 {
        u8 element_id;
        u8 length;
        __le16 key_id;
        u8 sequence_number[6];
        u8 mic[16];
} __packed;

/* Management MIC information element (IEEE 802.11w) for all variants */
struct ieee80211_mmie_var {
        u8 element_id;
        u8 length;
        __le16 key_id;
        u8 sequence_number[6];
        u8 mic[]; /* 8 or 16 bytes */
} __packed;

struct ieee80211_vendor_ie {
        u8 element_id;
        u8 len;
        u8 oui[3];
        u8 oui_type;
} __packed;

struct ieee80211_wmm_ac_param {
        u8 aci_aifsn; /* AIFSN, ACM, ACI */
        u8 cw; /* ECWmin, ECWmax (CW = 2^ECW - 1) */
        __le16 txop_limit;
} __packed;

struct ieee80211_wmm_param_ie {
        u8 element_id; /* Element ID: 221 (0xdd); */
        u8 len; /* Length: 24 */
        /* required fields for WMM version 1 */
        u8 oui[3]; /* 00:50:f2 */
        u8 oui_type; /* 2 */
        u8 oui_subtype; /* 1 */
        u8 version; /* 1 for WMM version 1.0 */
        u8 qos_info; /* AP/STA specific QoS info */
        u8 reserved; /* 0 */
        /* AC_BE, AC_BK, AC_VI, AC_VO */
        struct ieee80211_wmm_ac_param ac[4];
} __packed;

/* Control frames */
struct ieee80211_rts {
        __le16 frame_control;
        __le16 duration;
        u8 ra[ETH_ALEN];
        u8 ta[ETH_ALEN];
} __packed __aligned(2);

struct ieee80211_cts {
        __le16 frame_control;
        __le16 duration;
        u8 ra[ETH_ALEN];
} __packed __aligned(2);

struct ieee80211_pspoll {
        __le16 frame_control;
        __le16 aid;
        u8 bssid[ETH_ALEN];
        u8 ta[ETH_ALEN];
} __packed __aligned(2);

/* TDLS */

/* Channel switch timing */
struct ieee80211_ch_switch_timing {
        __le16 switch_time;
        __le16 switch_timeout;
} __packed;

/* Link-id information element */
struct ieee80211_tdls_lnkie {
        u8 ie_type; /* Link Identifier IE */
        u8 ie_len;
        u8 bssid[ETH_ALEN];
        u8 init_sta[ETH_ALEN];
        u8 resp_sta[ETH_ALEN];
} __packed;

struct ieee80211_tdls_data {
        u8 da[ETH_ALEN];
        u8 sa[ETH_ALEN];
        __be16 ether_type;
        u8 payload_type;
        u8 category;
        u8 action_code;
        union {
                struct {
                        u8 dialog_token;
                        __le16 capability;
                        u8 variable[];
                } __packed setup_req;
                struct {
                        __le16 status_code;
                        u8 dialog_token;
                        __le16 capability;
                        u8 variable[];
                } __packed setup_resp;
                struct {
                        __le16 status_code;
                        u8 dialog_token;
                        u8 variable[];
                } __packed setup_cfm;
                struct {
                        __le16 reason_code;
                        u8 variable[];
                } __packed teardown;
                struct {
                        u8 dialog_token;
                        u8 variable[];
                } __packed discover_req;
                struct {
                        u8 target_channel;
                        u8 oper_class;
                        u8 variable[];
                } __packed chan_switch_req;
                struct {
                        __le16 status_code;
                        u8 variable[];
                } __packed chan_switch_resp;
        } u;
} __packed;

/* Authentication algorithms */
#define WLAN_AUTH_OPEN 0
#define WLAN_AUTH_SHARED_KEY 1
#define WLAN_AUTH_FT 2
#define WLAN_AUTH_SAE 3
#define WLAN_AUTH_FILS_SK 4
#define WLAN_AUTH_FILS_SK_PFS 5
#define WLAN_AUTH_FILS_PK 6
#define WLAN_AUTH_IEEE8021X 8
#define WLAN_AUTH_EPPKE 9
#define WLAN_AUTH_LEAP 128

#define WLAN_AUTH_CHALLENGE_LEN 128

#define WLAN_CAPABILITY_ESS                (1<<0)
#define WLAN_CAPABILITY_IBSS                (1<<1)

/*
 * A mesh STA sets the ESS and IBSS capability bits to zero.
 * however, this holds true for p2p probe responses (in the p2p_find
 * phase) as well.
 */
#define WLAN_CAPABILITY_IS_STA_BSS(cap)        \
        (!((cap) & (WLAN_CAPABILITY_ESS | WLAN_CAPABILITY_IBSS)))

#define WLAN_CAPABILITY_CF_POLLABLE        (1<<2)
#define WLAN_CAPABILITY_CF_POLL_REQUEST        (1<<3)
#define WLAN_CAPABILITY_PRIVACY                (1<<4)
#define WLAN_CAPABILITY_SHORT_PREAMBLE        (1<<5)
#define WLAN_CAPABILITY_PBCC                (1<<6)
#define WLAN_CAPABILITY_CHANNEL_AGILITY        (1<<7)

/* 802.11h */
#define WLAN_CAPABILITY_SPECTRUM_MGMT        (1<<8)
#define WLAN_CAPABILITY_QOS                (1<<9)
#define WLAN_CAPABILITY_SHORT_SLOT_TIME        (1<<10)
#define WLAN_CAPABILITY_APSD                (1<<11)
#define WLAN_CAPABILITY_RADIO_MEASURE        (1<<12)
#define WLAN_CAPABILITY_DSSS_OFDM        (1<<13)
#define WLAN_CAPABILITY_DEL_BACK        (1<<14)
#define WLAN_CAPABILITY_IMM_BACK        (1<<15)

/* DMG (60gHz) 802.11ad */
/* type - bits 0..1 */
#define WLAN_CAPABILITY_DMG_TYPE_MASK                (3<<0)
#define WLAN_CAPABILITY_DMG_TYPE_IBSS                (1<<0) /* Tx by: STA */
#define WLAN_CAPABILITY_DMG_TYPE_PBSS                (2<<0) /* Tx by: PCP */
#define WLAN_CAPABILITY_DMG_TYPE_AP                (3<<0) /* Tx by: AP */

#define WLAN_CAPABILITY_DMG_CBAP_ONLY                (1<<2)
#define WLAN_CAPABILITY_DMG_CBAP_SOURCE                (1<<3)
#define WLAN_CAPABILITY_DMG_PRIVACY                (1<<4)
#define WLAN_CAPABILITY_DMG_ECPAC                (1<<5)

#define WLAN_CAPABILITY_DMG_SPECTRUM_MGMT        (1<<8)
#define WLAN_CAPABILITY_DMG_RADIO_MEASURE        (1<<12)

/* measurement */
#define IEEE80211_SPCT_MSR_RPRT_MODE_LATE        (1<<0)
#define IEEE80211_SPCT_MSR_RPRT_MODE_INCAPABLE        (1<<1)
#define IEEE80211_SPCT_MSR_RPRT_MODE_REFUSED        (1<<2)

#define IEEE80211_SPCT_MSR_RPRT_TYPE_BASIC        0
#define IEEE80211_SPCT_MSR_RPRT_TYPE_CCA        1
#define IEEE80211_SPCT_MSR_RPRT_TYPE_RPI        2
#define IEEE80211_SPCT_MSR_RPRT_TYPE_LCI        8
#define IEEE80211_SPCT_MSR_RPRT_TYPE_CIVIC        11

/* 802.11g ERP information element */
#define WLAN_ERP_NON_ERP_PRESENT (1<<0)
#define WLAN_ERP_USE_PROTECTION (1<<1)
#define WLAN_ERP_BARKER_PREAMBLE (1<<2)

/* WLAN_ERP_BARKER_PREAMBLE values */
enum {
        WLAN_ERP_PREAMBLE_SHORT = 0,
        WLAN_ERP_PREAMBLE_LONG = 1,
};

/* Band ID, 802.11ad #8.4.1.45 */
enum {
        IEEE80211_BANDID_TV_WS = 0, /* TV white spaces */
        IEEE80211_BANDID_SUB1  = 1, /* Sub-1 GHz (excluding TV white spaces) */
        IEEE80211_BANDID_2G    = 2, /* 2.4 GHz */
        IEEE80211_BANDID_3G    = 3, /* 3.6 GHz */
        IEEE80211_BANDID_5G    = 4, /* 4.9 and 5 GHz */
        IEEE80211_BANDID_60G   = 5, /* 60 GHz */
};

/* Status codes */
enum ieee80211_statuscode {
        WLAN_STATUS_SUCCESS = 0,
        WLAN_STATUS_UNSPECIFIED_FAILURE = 1,
        WLAN_STATUS_CAPS_UNSUPPORTED = 10,
        WLAN_STATUS_REASSOC_NO_ASSOC = 11,
        WLAN_STATUS_ASSOC_DENIED_UNSPEC = 12,
        WLAN_STATUS_NOT_SUPPORTED_AUTH_ALG = 13,
        WLAN_STATUS_UNKNOWN_AUTH_TRANSACTION = 14,
        WLAN_STATUS_CHALLENGE_FAIL = 15,
        WLAN_STATUS_AUTH_TIMEOUT = 16,
        WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA = 17,
        WLAN_STATUS_ASSOC_DENIED_RATES = 18,
        /* 802.11b */
        WLAN_STATUS_ASSOC_DENIED_NOSHORTPREAMBLE = 19,
        WLAN_STATUS_ASSOC_DENIED_NOPBCC = 20,
        WLAN_STATUS_ASSOC_DENIED_NOAGILITY = 21,
        /* 802.11h */
        WLAN_STATUS_ASSOC_DENIED_NOSPECTRUM = 22,
        WLAN_STATUS_ASSOC_REJECTED_BAD_POWER = 23,
        WLAN_STATUS_ASSOC_REJECTED_BAD_SUPP_CHAN = 24,
        /* 802.11g */
        WLAN_STATUS_ASSOC_DENIED_NOSHORTTIME = 25,
        WLAN_STATUS_ASSOC_DENIED_NODSSSOFDM = 26,
        /* 802.11w */
        WLAN_STATUS_ASSOC_REJECTED_TEMPORARILY = 30,
        WLAN_STATUS_ROBUST_MGMT_FRAME_POLICY_VIOLATION = 31,
        /* 802.11i */
        WLAN_STATUS_INVALID_IE = 40,
        WLAN_STATUS_INVALID_GROUP_CIPHER = 41,
        WLAN_STATUS_INVALID_PAIRWISE_CIPHER = 42,
        WLAN_STATUS_INVALID_AKMP = 43,
        WLAN_STATUS_UNSUPP_RSN_VERSION = 44,
        WLAN_STATUS_INVALID_RSN_IE_CAP = 45,
        WLAN_STATUS_CIPHER_SUITE_REJECTED = 46,
        /* 802.11e */
        WLAN_STATUS_UNSPECIFIED_QOS = 32,
        WLAN_STATUS_ASSOC_DENIED_NOBANDWIDTH = 33,
        WLAN_STATUS_ASSOC_DENIED_LOWACK = 34,
        WLAN_STATUS_ASSOC_DENIED_UNSUPP_QOS = 35,
        WLAN_STATUS_REQUEST_DECLINED = 37,
        WLAN_STATUS_INVALID_QOS_PARAM = 38,
        WLAN_STATUS_CHANGE_TSPEC = 39,
        WLAN_STATUS_WAIT_TS_DELAY = 47,
        WLAN_STATUS_NO_DIRECT_LINK = 48,
        WLAN_STATUS_STA_NOT_PRESENT = 49,
        WLAN_STATUS_STA_NOT_QSTA = 50,
        /* 802.11s */
        WLAN_STATUS_ANTI_CLOG_REQUIRED = 76,
        WLAN_STATUS_FCG_NOT_SUPP = 78,
        WLAN_STATUS_STA_NO_TBTT = 78,
        /* 802.11ad */
        WLAN_STATUS_REJECTED_WITH_SUGGESTED_CHANGES = 39,
        WLAN_STATUS_REJECTED_FOR_DELAY_PERIOD = 47,
        WLAN_STATUS_REJECT_WITH_SCHEDULE = 83,
        WLAN_STATUS_PENDING_ADMITTING_FST_SESSION = 86,
        WLAN_STATUS_PERFORMING_FST_NOW = 87,
        WLAN_STATUS_PENDING_GAP_IN_BA_WINDOW = 88,
        WLAN_STATUS_REJECT_U_PID_SETTING = 89,
        WLAN_STATUS_REJECT_DSE_BAND = 96,
        WLAN_STATUS_DENIED_WITH_SUGGESTED_BAND_AND_CHANNEL = 99,
        WLAN_STATUS_DENIED_DUE_TO_SPECTRUM_MANAGEMENT = 103,
        /* 802.11ah */
        WLAN_STATUS_REJECTED_NDP_BLOCK_ACK_SUGGESTED = 109,
        /* 802.11ai */
        WLAN_STATUS_FILS_AUTHENTICATION_FAILURE = 112,
        WLAN_STATUS_UNKNOWN_AUTHENTICATION_SERVER = 113,
        WLAN_STATUS_SAE_HASH_TO_ELEMENT = 126,
        WLAN_STATUS_SAE_PK = 127,
        WLAN_STATUS_DENIED_TID_TO_LINK_MAPPING = 133,
        WLAN_STATUS_PREF_TID_TO_LINK_MAPPING_SUGGESTED = 134,
        WLAN_STATUS_8021X_AUTH_SUCCESS = 153,
};


/* Reason codes */
enum ieee80211_reasoncode {
        WLAN_REASON_UNSPECIFIED = 1,
        WLAN_REASON_PREV_AUTH_NOT_VALID = 2,
        WLAN_REASON_DEAUTH_LEAVING = 3,
        WLAN_REASON_DISASSOC_DUE_TO_INACTIVITY = 4,
        WLAN_REASON_DISASSOC_AP_BUSY = 5,
        WLAN_REASON_CLASS2_FRAME_FROM_NONAUTH_STA = 6,
        WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA = 7,
        WLAN_REASON_DISASSOC_STA_HAS_LEFT = 8,
        WLAN_REASON_STA_REQ_ASSOC_WITHOUT_AUTH = 9,
        /* 802.11h */
        WLAN_REASON_DISASSOC_BAD_POWER = 10,
        WLAN_REASON_DISASSOC_BAD_SUPP_CHAN = 11,
        /* 802.11i */
        WLAN_REASON_INVALID_IE = 13,
        WLAN_REASON_MIC_FAILURE = 14,
        WLAN_REASON_4WAY_HANDSHAKE_TIMEOUT = 15,
        WLAN_REASON_GROUP_KEY_HANDSHAKE_TIMEOUT = 16,
        WLAN_REASON_IE_DIFFERENT = 17,
        WLAN_REASON_INVALID_GROUP_CIPHER = 18,
        WLAN_REASON_INVALID_PAIRWISE_CIPHER = 19,
        WLAN_REASON_INVALID_AKMP = 20,
        WLAN_REASON_UNSUPP_RSN_VERSION = 21,
        WLAN_REASON_INVALID_RSN_IE_CAP = 22,
        WLAN_REASON_IEEE8021X_FAILED = 23,
        WLAN_REASON_CIPHER_SUITE_REJECTED = 24,
        /* TDLS (802.11z) */
        WLAN_REASON_TDLS_TEARDOWN_UNREACHABLE = 25,
        WLAN_REASON_TDLS_TEARDOWN_UNSPECIFIED = 26,
        /* 802.11e */
        WLAN_REASON_DISASSOC_UNSPECIFIED_QOS = 32,
        WLAN_REASON_DISASSOC_QAP_NO_BANDWIDTH = 33,
        WLAN_REASON_DISASSOC_LOW_ACK = 34,
        WLAN_REASON_DISASSOC_QAP_EXCEED_TXOP = 35,
        WLAN_REASON_QSTA_LEAVE_QBSS = 36,
        WLAN_REASON_QSTA_NOT_USE = 37,
        WLAN_REASON_QSTA_REQUIRE_SETUP = 38,
        WLAN_REASON_QSTA_TIMEOUT = 39,
        WLAN_REASON_QSTA_CIPHER_NOT_SUPP = 45,
        /* 802.11s */
        WLAN_REASON_MESH_PEER_CANCELED = 52,
        WLAN_REASON_MESH_MAX_PEERS = 53,
        WLAN_REASON_MESH_CONFIG = 54,
        WLAN_REASON_MESH_CLOSE = 55,
        WLAN_REASON_MESH_MAX_RETRIES = 56,
        WLAN_REASON_MESH_CONFIRM_TIMEOUT = 57,
        WLAN_REASON_MESH_INVALID_GTK = 58,
        WLAN_REASON_MESH_INCONSISTENT_PARAM = 59,
        WLAN_REASON_MESH_INVALID_SECURITY = 60,
        WLAN_REASON_MESH_PATH_ERROR = 61,
        WLAN_REASON_MESH_PATH_NOFORWARD = 62,
        WLAN_REASON_MESH_PATH_DEST_UNREACHABLE = 63,
        WLAN_REASON_MAC_EXISTS_IN_MBSS = 64,
        WLAN_REASON_MESH_CHAN_REGULATORY = 65,
        WLAN_REASON_MESH_CHAN = 66,
};


/* Information Element IDs */
enum ieee80211_eid {
        WLAN_EID_SSID = 0,
        WLAN_EID_SUPP_RATES = 1,
        WLAN_EID_FH_PARAMS = 2, /* reserved now */
        WLAN_EID_DS_PARAMS = 3,
        WLAN_EID_CF_PARAMS = 4,
        WLAN_EID_TIM = 5,
        WLAN_EID_IBSS_PARAMS = 6,
        WLAN_EID_COUNTRY = 7,
        /* 8, 9 reserved */
        WLAN_EID_REQUEST = 10,
        WLAN_EID_QBSS_LOAD = 11,
        WLAN_EID_EDCA_PARAM_SET = 12,
        WLAN_EID_TSPEC = 13,
        WLAN_EID_TCLAS = 14,
        WLAN_EID_SCHEDULE = 15,
        WLAN_EID_CHALLENGE = 16,
        /* 17-31 reserved for challenge text extension */
        WLAN_EID_PWR_CONSTRAINT = 32,
        WLAN_EID_PWR_CAPABILITY = 33,
        WLAN_EID_TPC_REQUEST = 34,
        WLAN_EID_TPC_REPORT = 35,
        WLAN_EID_SUPPORTED_CHANNELS = 36,
        WLAN_EID_CHANNEL_SWITCH = 37,
        WLAN_EID_MEASURE_REQUEST = 38,
        WLAN_EID_MEASURE_REPORT = 39,
        WLAN_EID_QUIET = 40,
        WLAN_EID_IBSS_DFS = 41,
        WLAN_EID_ERP_INFO = 42,
        WLAN_EID_TS_DELAY = 43,
        WLAN_EID_TCLAS_PROCESSING = 44,
        WLAN_EID_HT_CAPABILITY = 45,
        WLAN_EID_QOS_CAPA = 46,
        /* 47 reserved for Broadcom */
        WLAN_EID_RSN = 48,
        WLAN_EID_802_15_COEX = 49,
        WLAN_EID_EXT_SUPP_RATES = 50,
        WLAN_EID_AP_CHAN_REPORT = 51,
        WLAN_EID_NEIGHBOR_REPORT = 52,
        WLAN_EID_RCPI = 53,
        WLAN_EID_MOBILITY_DOMAIN = 54,
        WLAN_EID_FAST_BSS_TRANSITION = 55,
        WLAN_EID_TIMEOUT_INTERVAL = 56,
        WLAN_EID_RIC_DATA = 57,
        WLAN_EID_DSE_REGISTERED_LOCATION = 58,
        WLAN_EID_SUPPORTED_REGULATORY_CLASSES = 59,
        WLAN_EID_EXT_CHANSWITCH_ANN = 60,
        WLAN_EID_HT_OPERATION = 61,
        WLAN_EID_SECONDARY_CHANNEL_OFFSET = 62,
        WLAN_EID_BSS_AVG_ACCESS_DELAY = 63,
        WLAN_EID_ANTENNA_INFO = 64,
        WLAN_EID_RSNI = 65,
        WLAN_EID_MEASUREMENT_PILOT_TX_INFO = 66,
        WLAN_EID_BSS_AVAILABLE_CAPACITY = 67,
        WLAN_EID_BSS_AC_ACCESS_DELAY = 68,
        WLAN_EID_TIME_ADVERTISEMENT = 69,
        WLAN_EID_RRM_ENABLED_CAPABILITIES = 70,
        WLAN_EID_MULTIPLE_BSSID = 71,
        WLAN_EID_BSS_COEX_2040 = 72,
        WLAN_EID_BSS_INTOLERANT_CHL_REPORT = 73,
        WLAN_EID_OVERLAP_BSS_SCAN_PARAM = 74,
        WLAN_EID_RIC_DESCRIPTOR = 75,
        WLAN_EID_MMIE = 76,
        WLAN_EID_ASSOC_COMEBACK_TIME = 77,
        WLAN_EID_EVENT_REQUEST = 78,
        WLAN_EID_EVENT_REPORT = 79,
        WLAN_EID_DIAGNOSTIC_REQUEST = 80,
        WLAN_EID_DIAGNOSTIC_REPORT = 81,
        WLAN_EID_LOCATION_PARAMS = 82,
        WLAN_EID_NON_TX_BSSID_CAP =  83,
        WLAN_EID_SSID_LIST = 84,
        WLAN_EID_MULTI_BSSID_IDX = 85,
        WLAN_EID_FMS_DESCRIPTOR = 86,
        WLAN_EID_FMS_REQUEST = 87,
        WLAN_EID_FMS_RESPONSE = 88,
        WLAN_EID_QOS_TRAFFIC_CAPA = 89,
        WLAN_EID_BSS_MAX_IDLE_PERIOD = 90,
        WLAN_EID_TSF_REQUEST = 91,
        WLAN_EID_TSF_RESPOSNE = 92,
        WLAN_EID_WNM_SLEEP_MODE = 93,
        WLAN_EID_TIM_BCAST_REQ = 94,
        WLAN_EID_TIM_BCAST_RESP = 95,
        WLAN_EID_COLL_IF_REPORT = 96,
        WLAN_EID_CHANNEL_USAGE = 97,
        WLAN_EID_TIME_ZONE = 98,
        WLAN_EID_DMS_REQUEST = 99,
        WLAN_EID_DMS_RESPONSE = 100,
        WLAN_EID_LINK_ID = 101,
        WLAN_EID_WAKEUP_SCHEDUL = 102,
        /* 103 reserved */
        WLAN_EID_CHAN_SWITCH_TIMING = 104,
        WLAN_EID_PTI_CONTROL = 105,
        WLAN_EID_PU_BUFFER_STATUS = 106,
        WLAN_EID_INTERWORKING = 107,
        WLAN_EID_ADVERTISEMENT_PROTOCOL = 108,
        WLAN_EID_EXPEDITED_BW_REQ = 109,
        WLAN_EID_QOS_MAP_SET = 110,
        WLAN_EID_ROAMING_CONSORTIUM = 111,
        WLAN_EID_EMERGENCY_ALERT = 112,
        WLAN_EID_MESH_CONFIG = 113,
        WLAN_EID_MESH_ID = 114,
        WLAN_EID_LINK_METRIC_REPORT = 115,
        WLAN_EID_CONGESTION_NOTIFICATION = 116,
        WLAN_EID_PEER_MGMT = 117,
        WLAN_EID_CHAN_SWITCH_PARAM = 118,
        WLAN_EID_MESH_AWAKE_WINDOW = 119,
        WLAN_EID_BEACON_TIMING = 120,
        WLAN_EID_MCCAOP_SETUP_REQ = 121,
        WLAN_EID_MCCAOP_SETUP_RESP = 122,
        WLAN_EID_MCCAOP_ADVERT = 123,
        WLAN_EID_MCCAOP_TEARDOWN = 124,
        WLAN_EID_GANN = 125,
        WLAN_EID_RANN = 126,
        WLAN_EID_EXT_CAPABILITY = 127,
        /* 128, 129 reserved for Agere */
        WLAN_EID_PREQ = 130,
        WLAN_EID_PREP = 131,
        WLAN_EID_PERR = 132,
        /* 133-136 reserved for Cisco */
        WLAN_EID_PXU = 137,
        WLAN_EID_PXUC = 138,
        WLAN_EID_AUTH_MESH_PEER_EXCH = 139,
        WLAN_EID_MIC = 140,
        WLAN_EID_DESTINATION_URI = 141,
        WLAN_EID_UAPSD_COEX = 142,
        WLAN_EID_WAKEUP_SCHEDULE = 143,
        WLAN_EID_EXT_SCHEDULE = 144,
        WLAN_EID_STA_AVAILABILITY = 145,
        WLAN_EID_DMG_TSPEC = 146,
        WLAN_EID_DMG_AT = 147,
        WLAN_EID_DMG_CAP = 148,
        /* 149 reserved for Cisco */
        WLAN_EID_CISCO_VENDOR_SPECIFIC = 150,
        WLAN_EID_DMG_OPERATION = 151,
        WLAN_EID_DMG_BSS_PARAM_CHANGE = 152,
        WLAN_EID_DMG_BEAM_REFINEMENT = 153,
        WLAN_EID_CHANNEL_MEASURE_FEEDBACK = 154,
        /* 155-156 reserved for Cisco */
        WLAN_EID_AWAKE_WINDOW = 157,
        WLAN_EID_MULTI_BAND = 158,
        WLAN_EID_ADDBA_EXT = 159,
        WLAN_EID_NEXT_PCP_LIST = 160,
        WLAN_EID_PCP_HANDOVER = 161,
        WLAN_EID_DMG_LINK_MARGIN = 162,
        WLAN_EID_SWITCHING_STREAM = 163,
        WLAN_EID_SESSION_TRANSITION = 164,
        WLAN_EID_DYN_TONE_PAIRING_REPORT = 165,
        WLAN_EID_CLUSTER_REPORT = 166,
        WLAN_EID_RELAY_CAP = 167,
        WLAN_EID_RELAY_XFER_PARAM_SET = 168,
        WLAN_EID_BEAM_LINK_MAINT = 169,
        WLAN_EID_MULTIPLE_MAC_ADDR = 170,
        WLAN_EID_U_PID = 171,
        WLAN_EID_DMG_LINK_ADAPT_ACK = 172,
        /* 173 reserved for Symbol */
        WLAN_EID_MCCAOP_ADV_OVERVIEW = 174,
        WLAN_EID_QUIET_PERIOD_REQ = 175,
        /* 176 reserved for Symbol */
        WLAN_EID_QUIET_PERIOD_RESP = 177,
        /* 178-179 reserved for Symbol */
        /* 180 reserved for ISO/IEC 20011 */
        WLAN_EID_EPAC_POLICY = 182,
        WLAN_EID_CLISTER_TIME_OFF = 183,
        WLAN_EID_INTER_AC_PRIO = 184,
        WLAN_EID_SCS_DESCRIPTOR = 185,
        WLAN_EID_QLOAD_REPORT = 186,
        WLAN_EID_HCCA_TXOP_UPDATE_COUNT = 187,
        WLAN_EID_HL_STREAM_ID = 188,
        WLAN_EID_GCR_GROUP_ADDR = 189,
        WLAN_EID_ANTENNA_SECTOR_ID_PATTERN = 190,
        WLAN_EID_VHT_CAPABILITY = 191,
        WLAN_EID_VHT_OPERATION = 192,
        WLAN_EID_EXTENDED_BSS_LOAD = 193,
        WLAN_EID_WIDE_BW_CHANNEL_SWITCH = 194,
        WLAN_EID_TX_POWER_ENVELOPE = 195,
        WLAN_EID_CHANNEL_SWITCH_WRAPPER = 196,
        WLAN_EID_AID = 197,
        WLAN_EID_QUIET_CHANNEL = 198,
        WLAN_EID_OPMODE_NOTIF = 199,

        WLAN_EID_REDUCED_NEIGHBOR_REPORT = 201,

        WLAN_EID_AID_REQUEST = 210,
        WLAN_EID_AID_RESPONSE = 211,
        WLAN_EID_S1G_BCN_COMPAT = 213,
        WLAN_EID_S1G_SHORT_BCN_INTERVAL = 214,
        WLAN_EID_S1G_TWT = 216,
        WLAN_EID_S1G_CAPABILITIES = 217,
        WLAN_EID_VENDOR_SPECIFIC = 221,
        WLAN_EID_QOS_PARAMETER = 222,
        WLAN_EID_S1G_OPERATION = 232,
        WLAN_EID_CAG_NUMBER = 237,
        WLAN_EID_AP_CSN = 239,
        WLAN_EID_FILS_INDICATION = 240,
        WLAN_EID_DILS = 241,
        WLAN_EID_FRAGMENT = 242,
        WLAN_EID_RSNX = 244,
        WLAN_EID_EXTENSION = 255
};

/* Element ID Extensions for Element ID 255 */
enum ieee80211_eid_ext {
        WLAN_EID_EXT_ASSOC_DELAY_INFO = 1,
        WLAN_EID_EXT_FILS_REQ_PARAMS = 2,
        WLAN_EID_EXT_FILS_KEY_CONFIRM = 3,
        WLAN_EID_EXT_FILS_SESSION = 4,
        WLAN_EID_EXT_FILS_HLP_CONTAINER = 5,
        WLAN_EID_EXT_FILS_IP_ADDR_ASSIGN = 6,
        WLAN_EID_EXT_KEY_DELIVERY = 7,
        WLAN_EID_EXT_FILS_WRAPPED_DATA = 8,
        WLAN_EID_EXT_FILS_PUBLIC_KEY = 12,
        WLAN_EID_EXT_FILS_NONCE = 13,
        WLAN_EID_EXT_FUTURE_CHAN_GUIDANCE = 14,
        WLAN_EID_EXT_DH_PARAMETER = 32,
        WLAN_EID_EXT_HE_CAPABILITY = 35,
        WLAN_EID_EXT_HE_OPERATION = 36,
        WLAN_EID_EXT_UORA = 37,
        WLAN_EID_EXT_HE_MU_EDCA = 38,
        WLAN_EID_EXT_HE_SPR = 39,
        WLAN_EID_EXT_NDP_FEEDBACK_REPORT_PARAMSET = 41,
        WLAN_EID_EXT_BSS_COLOR_CHG_ANN = 42,
        WLAN_EID_EXT_QUIET_TIME_PERIOD_SETUP = 43,
        WLAN_EID_EXT_ESS_REPORT = 45,
        WLAN_EID_EXT_OPS = 46,
        WLAN_EID_EXT_HE_BSS_LOAD = 47,
        WLAN_EID_EXT_MAX_CHANNEL_SWITCH_TIME = 52,
        WLAN_EID_EXT_MULTIPLE_BSSID_CONFIGURATION = 55,
        WLAN_EID_EXT_NON_INHERITANCE = 56,
        WLAN_EID_EXT_KNOWN_BSSID = 57,
        WLAN_EID_EXT_SHORT_SSID_LIST = 58,
        WLAN_EID_EXT_HE_6GHZ_CAPA = 59,
        WLAN_EID_EXT_UL_MU_POWER_CAPA = 60,
        WLAN_EID_EXT_EHT_OPERATION = 106,
        WLAN_EID_EXT_EHT_MULTI_LINK = 107,
        WLAN_EID_EXT_EHT_CAPABILITY = 108,
        WLAN_EID_EXT_TID_TO_LINK_MAPPING = 109,
        WLAN_EID_EXT_BANDWIDTH_INDICATION = 135,
        WLAN_EID_EXT_KNOWN_STA_IDENTIFCATION = 136,
        WLAN_EID_EXT_NON_AP_STA_REG_CON = 137,
        WLAN_EID_EXT_UHR_OPER = 151,
        WLAN_EID_EXT_UHR_CAPA = 152,
        WLAN_EID_EXT_MACP = 153,
        WLAN_EID_EXT_SMD = 154,
        WLAN_EID_EXT_BSS_SMD_TRANS_PARAMS = 155,
        WLAN_EID_EXT_CHAN_USAGE = 156,
        WLAN_EID_EXT_UHR_MODE_CHG = 157,
        WLAN_EID_EXT_UHR_PARAM_UPD = 158,
        WLAN_EID_EXT_TXPI = 159,
};

/* Action category code */
enum ieee80211_category {
        WLAN_CATEGORY_SPECTRUM_MGMT = 0,
        WLAN_CATEGORY_QOS = 1,
        WLAN_CATEGORY_DLS = 2,
        WLAN_CATEGORY_BACK = 3,
        WLAN_CATEGORY_PUBLIC = 4,
        WLAN_CATEGORY_RADIO_MEASUREMENT = 5,
        WLAN_CATEGORY_FAST_BBS_TRANSITION = 6,
        WLAN_CATEGORY_HT = 7,
        WLAN_CATEGORY_SA_QUERY = 8,
        WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION = 9,
        WLAN_CATEGORY_WNM = 10,
        WLAN_CATEGORY_WNM_UNPROTECTED = 11,
        WLAN_CATEGORY_TDLS = 12,
        WLAN_CATEGORY_MESH_ACTION = 13,
        WLAN_CATEGORY_MULTIHOP_ACTION = 14,
        WLAN_CATEGORY_SELF_PROTECTED = 15,
        WLAN_CATEGORY_DMG = 16,
        WLAN_CATEGORY_WMM = 17,
        WLAN_CATEGORY_FST = 18,
        WLAN_CATEGORY_UNPROT_DMG = 20,
        WLAN_CATEGORY_VHT = 21,
        WLAN_CATEGORY_S1G = 22,
        WLAN_CATEGORY_PROTECTED_EHT = 37,
        WLAN_CATEGORY_VENDOR_SPECIFIC_PROTECTED = 126,
        WLAN_CATEGORY_VENDOR_SPECIFIC = 127,
};

/* SPECTRUM_MGMT action code */
enum ieee80211_spectrum_mgmt_actioncode {
        WLAN_ACTION_SPCT_MSR_REQ = 0,
        WLAN_ACTION_SPCT_MSR_RPRT = 1,
        WLAN_ACTION_SPCT_TPC_REQ = 2,
        WLAN_ACTION_SPCT_TPC_RPRT = 3,
        WLAN_ACTION_SPCT_CHL_SWITCH = 4,
};

/* Self Protected Action codes */
enum ieee80211_self_protected_actioncode {
        WLAN_SP_RESERVED = 0,
        WLAN_SP_MESH_PEERING_OPEN = 1,
        WLAN_SP_MESH_PEERING_CONFIRM = 2,
        WLAN_SP_MESH_PEERING_CLOSE = 3,
        WLAN_SP_MGK_INFORM = 4,
        WLAN_SP_MGK_ACK = 5,
};

/* Unprotected WNM action codes */
enum ieee80211_unprotected_wnm_actioncode {
        WLAN_UNPROTECTED_WNM_ACTION_TIM = 0,
        WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE = 1,
};

/* Security key length */
enum ieee80211_key_len {
        WLAN_KEY_LEN_WEP40 = 5,
        WLAN_KEY_LEN_WEP104 = 13,
        WLAN_KEY_LEN_CCMP = 16,
        WLAN_KEY_LEN_CCMP_256 = 32,
        WLAN_KEY_LEN_TKIP = 32,
        WLAN_KEY_LEN_AES_CMAC = 16,
        WLAN_KEY_LEN_SMS4 = 32,
        WLAN_KEY_LEN_GCMP = 16,
        WLAN_KEY_LEN_GCMP_256 = 32,
        WLAN_KEY_LEN_BIP_CMAC_256 = 32,
        WLAN_KEY_LEN_BIP_GMAC_128 = 16,
        WLAN_KEY_LEN_BIP_GMAC_256 = 32,
};

/* Radio measurement action codes as defined in IEEE 802.11-2024 - Table 9-470 */
enum ieee80211_radio_measurement_actioncode {
        WLAN_RM_ACTION_RADIO_MEASUREMENT_REQUEST = 0,
        WLAN_RM_ACTION_RADIO_MEASUREMENT_REPORT  = 1,
        WLAN_RM_ACTION_LINK_MEASUREMENT_REQUEST  = 2,
        WLAN_RM_ACTION_LINK_MEASUREMENT_REPORT   = 3,
        WLAN_RM_ACTION_NEIGHBOR_REPORT_REQUEST   = 4,
        WLAN_RM_ACTION_NEIGHBOR_REPORT_RESPONSE  = 5,
};

#define IEEE80211_WEP_IV_LEN                4
#define IEEE80211_WEP_ICV_LEN                4
#define IEEE80211_CCMP_HDR_LEN                8
#define IEEE80211_CCMP_MIC_LEN                8
#define IEEE80211_CCMP_PN_LEN                6
#define IEEE80211_CCMP_256_HDR_LEN        8
#define IEEE80211_CCMP_256_MIC_LEN        16
#define IEEE80211_CCMP_256_PN_LEN        6
#define IEEE80211_TKIP_IV_LEN                8
#define IEEE80211_TKIP_ICV_LEN                4
#define IEEE80211_CMAC_PN_LEN                6
#define IEEE80211_GMAC_PN_LEN                6
#define IEEE80211_GCMP_HDR_LEN                8
#define IEEE80211_GCMP_MIC_LEN                16
#define IEEE80211_GCMP_PN_LEN                6
#define IEEE80211_CMAC_128_MIC_LEN        8
#define IEEE80211_CMAC_256_MIC_LEN        16
#define IEEE80211_GMAC_MIC_LEN                16

#define FILS_NONCE_LEN                        16
#define FILS_MAX_KEK_LEN                64

#define FILS_ERP_MAX_USERNAME_LEN        16
#define FILS_ERP_MAX_REALM_LEN                253
#define FILS_ERP_MAX_RRK_LEN                64

#define PMK_MAX_LEN                        64
#define SAE_PASSWORD_MAX_LEN                128

#define MICHAEL_MIC_LEN                        8

void michael_mic(const u8 *key, struct ieee80211_hdr *hdr,
                 const u8 *data, size_t data_len, u8 *mic);

/* Public action codes (IEEE Std 802.11-2016, 9.6.8.1, Table 9-307) */
enum ieee80211_pub_actioncode {
        WLAN_PUB_ACTION_20_40_BSS_COEX = 0,
        WLAN_PUB_ACTION_DSE_ENABLEMENT = 1,
        WLAN_PUB_ACTION_DSE_DEENABLEMENT = 2,
        WLAN_PUB_ACTION_DSE_REG_LOC_ANN = 3,
        WLAN_PUB_ACTION_EXT_CHANSW_ANN = 4,
        WLAN_PUB_ACTION_DSE_MSMT_REQ = 5,
        WLAN_PUB_ACTION_DSE_MSMT_RESP = 6,
        WLAN_PUB_ACTION_MSMT_PILOT = 7,
        WLAN_PUB_ACTION_DSE_PC = 8,
        WLAN_PUB_ACTION_VENDOR_SPECIFIC = 9,
        WLAN_PUB_ACTION_GAS_INITIAL_REQ = 10,
        WLAN_PUB_ACTION_GAS_INITIAL_RESP = 11,
        WLAN_PUB_ACTION_GAS_COMEBACK_REQ = 12,
        WLAN_PUB_ACTION_GAS_COMEBACK_RESP = 13,
        WLAN_PUB_ACTION_TDLS_DISCOVER_RES = 14,
        WLAN_PUB_ACTION_LOC_TRACK_NOTI = 15,
        WLAN_PUB_ACTION_QAB_REQUEST_FRAME = 16,
        WLAN_PUB_ACTION_QAB_RESPONSE_FRAME = 17,
        WLAN_PUB_ACTION_QMF_POLICY = 18,
        WLAN_PUB_ACTION_QMF_POLICY_CHANGE = 19,
        WLAN_PUB_ACTION_QLOAD_REQUEST = 20,
        WLAN_PUB_ACTION_QLOAD_REPORT = 21,
        WLAN_PUB_ACTION_HCCA_TXOP_ADVERT = 22,
        WLAN_PUB_ACTION_HCCA_TXOP_RESPONSE = 23,
        WLAN_PUB_ACTION_PUBLIC_KEY = 24,
        WLAN_PUB_ACTION_CHANNEL_AVAIL_QUERY = 25,
        WLAN_PUB_ACTION_CHANNEL_SCHEDULE_MGMT = 26,
        WLAN_PUB_ACTION_CONTACT_VERI_SIGNAL = 27,
        WLAN_PUB_ACTION_GDD_ENABLEMENT_REQ = 28,
        WLAN_PUB_ACTION_GDD_ENABLEMENT_RESP = 29,
        WLAN_PUB_ACTION_NETWORK_CHANNEL_CONTROL = 30,
        WLAN_PUB_ACTION_WHITE_SPACE_MAP_ANN = 31,
        WLAN_PUB_ACTION_FTM_REQUEST = 32,
        WLAN_PUB_ACTION_FTM_RESPONSE = 33,
        WLAN_PUB_ACTION_FILS_DISCOVERY = 34,
};

/* TDLS action codes */
enum ieee80211_tdls_actioncode {
        WLAN_TDLS_SETUP_REQUEST = 0,
        WLAN_TDLS_SETUP_RESPONSE = 1,
        WLAN_TDLS_SETUP_CONFIRM = 2,
        WLAN_TDLS_TEARDOWN = 3,
        WLAN_TDLS_PEER_TRAFFIC_INDICATION = 4,
        WLAN_TDLS_CHANNEL_SWITCH_REQUEST = 5,
        WLAN_TDLS_CHANNEL_SWITCH_RESPONSE = 6,
        WLAN_TDLS_PEER_PSM_REQUEST = 7,
        WLAN_TDLS_PEER_PSM_RESPONSE = 8,
        WLAN_TDLS_PEER_TRAFFIC_RESPONSE = 9,
        WLAN_TDLS_DISCOVERY_REQUEST = 10,
};

/* Extended Channel Switching capability to be set in the 1st byte of
 * the @WLAN_EID_EXT_CAPABILITY information element
 */
#define WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING        BIT(2)

/* Multiple BSSID capability is set in the 6th bit of 3rd byte of the
 * @WLAN_EID_EXT_CAPABILITY information element
 */
#define WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT        BIT(6)

/* Timing Measurement protocol for time sync is set in the 7th bit of 3rd byte
 * of the @WLAN_EID_EXT_CAPABILITY information element
 */
#define WLAN_EXT_CAPA3_TIMING_MEASUREMENT_SUPPORT        BIT(7)

/* TDLS capabilities in the 4th byte of @WLAN_EID_EXT_CAPABILITY */
#define WLAN_EXT_CAPA4_TDLS_BUFFER_STA                BIT(4)
#define WLAN_EXT_CAPA4_TDLS_PEER_PSM                BIT(5)
#define WLAN_EXT_CAPA4_TDLS_CHAN_SWITCH                BIT(6)

/* Interworking capabilities are set in 7th bit of 4th byte of the
 * @WLAN_EID_EXT_CAPABILITY information element
 */
#define WLAN_EXT_CAPA4_INTERWORKING_ENABLED        BIT(7)

/*
 * TDLS capabililites to be enabled in the 5th byte of the
 * @WLAN_EID_EXT_CAPABILITY information element
 */
#define WLAN_EXT_CAPA5_TDLS_ENABLED        BIT(5)
#define WLAN_EXT_CAPA5_TDLS_PROHIBITED        BIT(6)
#define WLAN_EXT_CAPA5_TDLS_CH_SW_PROHIBITED        BIT(7)

#define WLAN_EXT_CAPA8_TDLS_WIDE_BW_ENABLED        BIT(5)
#define WLAN_EXT_CAPA8_OPMODE_NOTIF        BIT(6)

/* Defines the maximal number of MSDUs in an A-MSDU. */
#define WLAN_EXT_CAPA8_MAX_MSDU_IN_AMSDU_LSB        BIT(7)
#define WLAN_EXT_CAPA9_MAX_MSDU_IN_AMSDU_MSB        BIT(0)

/*
 * Fine Timing Measurement Initiator - bit 71 of @WLAN_EID_EXT_CAPABILITY
 * information element
 */
#define WLAN_EXT_CAPA9_FTM_INITIATOR        BIT(7)

/* Defines support for TWT Requester and TWT Responder */
#define WLAN_EXT_CAPA10_TWT_REQUESTER_SUPPORT        BIT(5)
#define WLAN_EXT_CAPA10_TWT_RESPONDER_SUPPORT        BIT(6)

/*
 * When set, indicates that the AP is able to tolerate 26-tone RU UL
 * OFDMA transmissions using HE TB PPDU from OBSS (not falsely classify the
 * 26-tone RU UL OFDMA transmissions as radar pulses).
 */
#define WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT BIT(7)

/* Defines support for enhanced multi-bssid advertisement*/
#define WLAN_EXT_CAPA11_EMA_SUPPORT        BIT(3)

/* Enable Beacon Protection */
#define WLAN_EXT_CAPA11_BCN_PROTECT        BIT(4)

/* TDLS specific payload type in the LLC/SNAP header */
#define WLAN_TDLS_SNAP_RFTYPE        0x2

/* BSS Coex IE information field bits */
#define WLAN_BSS_COEX_INFORMATION_REQUEST        BIT(0)

/*
 * IEEE 802.11-2007 7.3.2.9 Country information element
 *
 * Minimum length is 8 octets, ie len must be evenly
 * divisible by 2
 */

/* Although the spec says 8 I'm seeing 6 in practice */
#define IEEE80211_COUNTRY_IE_MIN_LEN        6

/* The Country String field of the element shall be 3 octets in length */
#define IEEE80211_COUNTRY_STRING_LEN        3

/*
 * For regulatory extension stuff see IEEE 802.11-2007
 * Annex I (page 1141) and Annex J (page 1147). Also
 * review 7.3.2.9.
 *
 * When dot11RegulatoryClassesRequired is true and the
 * first_channel/reg_extension_id is >= 201 then the IE
 * compromises of the 'ext' struct represented below:
 *
 *  - Regulatory extension ID - when generating IE this just needs
 *    to be monotonically increasing for each triplet passed in
 *    the IE
 *  - Regulatory class - index into set of rules
 *  - Coverage class - index into air propagation time (Table 7-27),
 *    in microseconds, you can compute the air propagation time from
 *    the index by multiplying by 3, so index 10 yields a propagation
 *    of 10 us. Valid values are 0-31, values 32-255 are not defined
 *    yet. A value of 0 inicates air propagation of <= 1 us.
 *
 *  See also Table I.2 for Emission limit sets and table
 *  I.3 for Behavior limit sets. Table J.1 indicates how to map
 *  a reg_class to an emission limit set and behavior limit set.
 */
#define IEEE80211_COUNTRY_EXTENSION_ID 201

/*
 *  Channels numbers in the IE must be monotonically increasing
 *  if dot11RegulatoryClassesRequired is not true.
 *
 *  If dot11RegulatoryClassesRequired is true consecutive
 *  subband triplets following a regulatory triplet shall
 *  have monotonically increasing first_channel number fields.
 *
 *  Channel numbers shall not overlap.
 *
 *  Note that max_power is signed.
 */
struct ieee80211_country_ie_triplet {
        union {
                struct {
                        u8 first_channel;
                        u8 num_channels;
                        s8 max_power;
                } __packed chans;
                struct {
                        u8 reg_extension_id;
                        u8 reg_class;
                        u8 coverage_class;
                } __packed ext;
        };
} __packed;

enum ieee80211_timeout_interval_type {
        WLAN_TIMEOUT_REASSOC_DEADLINE = 1 /* 802.11r */,
        WLAN_TIMEOUT_KEY_LIFETIME = 2 /* 802.11r */,
        WLAN_TIMEOUT_ASSOC_COMEBACK = 3 /* 802.11w */,
};

/**
 * struct ieee80211_timeout_interval_ie - Timeout Interval element
 * @type: type, see &enum ieee80211_timeout_interval_type
 * @value: timeout interval value
 */
struct ieee80211_timeout_interval_ie {
        u8 type;
        __le32 value;
} __packed;

/**
 * enum ieee80211_idle_options - BSS idle options
 * @WLAN_IDLE_OPTIONS_PROTECTED_KEEP_ALIVE: the station should send an RSN
 *        protected frame to the AP to reset the idle timer at the AP for
 *        the station.
 */
enum ieee80211_idle_options {
        WLAN_IDLE_OPTIONS_PROTECTED_KEEP_ALIVE = BIT(0),
};

/**
 * struct ieee80211_bss_max_idle_period_ie - BSS max idle period element struct
 *
 * This structure refers to "BSS Max idle period element"
 *
 * @max_idle_period: indicates the time period during which a station can
 *        refrain from transmitting frames to its associated AP without being
 *        disassociated. In units of 1000 TUs.
 * @idle_options: indicates the options associated with the BSS idle capability
 *        as specified in &enum ieee80211_idle_options.
 */
struct ieee80211_bss_max_idle_period_ie {
        __le16 max_idle_period;
        u8 idle_options;
} __packed;

/* SA Query action */
enum ieee80211_sa_query_action {
        WLAN_ACTION_SA_QUERY_REQUEST = 0,
        WLAN_ACTION_SA_QUERY_RESPONSE = 1,
};

/**
 * struct ieee80211_bssid_index - multiple BSSID index element structure
 *
 * This structure refers to "Multiple BSSID-index element"
 *
 * @bssid_index: BSSID index
 * @dtim_period: optional, overrides transmitted BSS dtim period
 * @dtim_count: optional, overrides transmitted BSS dtim count
 */
struct ieee80211_bssid_index {
        u8 bssid_index;
        u8 dtim_period;
        u8 dtim_count;
};

/**
 * struct ieee80211_multiple_bssid_configuration - multiple BSSID configuration
 *        element structure
 *
 * This structure refers to "Multiple BSSID Configuration element"
 *
 * @bssid_count: total number of active BSSIDs in the set
 * @profile_periodicity: the least number of beacon frames need to be received
 *        in order to discover all the nontransmitted BSSIDs in the set.
 */
struct ieee80211_multiple_bssid_configuration {
        u8 bssid_count;
        u8 profile_periodicity;
};

#define SUITE(oui, id)        (((oui) << 8) | (id))

/* cipher suite selectors */
#define WLAN_CIPHER_SUITE_USE_GROUP        SUITE(0x000FAC, 0)
#define WLAN_CIPHER_SUITE_WEP40                SUITE(0x000FAC, 1)
#define WLAN_CIPHER_SUITE_TKIP                SUITE(0x000FAC, 2)
/* reserved:                                 SUITE(0x000FAC, 3) */
#define WLAN_CIPHER_SUITE_CCMP                SUITE(0x000FAC, 4)
#define WLAN_CIPHER_SUITE_WEP104        SUITE(0x000FAC, 5)
#define WLAN_CIPHER_SUITE_AES_CMAC        SUITE(0x000FAC, 6)
#define WLAN_CIPHER_SUITE_GCMP                SUITE(0x000FAC, 8)
#define WLAN_CIPHER_SUITE_GCMP_256        SUITE(0x000FAC, 9)
#define WLAN_CIPHER_SUITE_CCMP_256        SUITE(0x000FAC, 10)
#define WLAN_CIPHER_SUITE_BIP_GMAC_128        SUITE(0x000FAC, 11)
#define WLAN_CIPHER_SUITE_BIP_GMAC_256        SUITE(0x000FAC, 12)
#define WLAN_CIPHER_SUITE_BIP_CMAC_256        SUITE(0x000FAC, 13)

#define WLAN_CIPHER_SUITE_SMS4                SUITE(0x001472, 1)

/* AKM suite selectors */
#define WLAN_AKM_SUITE_8021X                        SUITE(0x000FAC, 1)
#define WLAN_AKM_SUITE_PSK                        SUITE(0x000FAC, 2)
#define WLAN_AKM_SUITE_FT_8021X                        SUITE(0x000FAC, 3)
#define WLAN_AKM_SUITE_FT_PSK                        SUITE(0x000FAC, 4)
#define WLAN_AKM_SUITE_8021X_SHA256                SUITE(0x000FAC, 5)
#define WLAN_AKM_SUITE_PSK_SHA256                SUITE(0x000FAC, 6)
#define WLAN_AKM_SUITE_TDLS                        SUITE(0x000FAC, 7)
#define WLAN_AKM_SUITE_SAE                        SUITE(0x000FAC, 8)
#define WLAN_AKM_SUITE_FT_OVER_SAE                SUITE(0x000FAC, 9)
#define WLAN_AKM_SUITE_AP_PEER_KEY                SUITE(0x000FAC, 10)
#define WLAN_AKM_SUITE_8021X_SUITE_B                SUITE(0x000FAC, 11)
#define WLAN_AKM_SUITE_8021X_SUITE_B_192        SUITE(0x000FAC, 12)
#define WLAN_AKM_SUITE_FT_8021X_SHA384                SUITE(0x000FAC, 13)
#define WLAN_AKM_SUITE_FILS_SHA256                SUITE(0x000FAC, 14)
#define WLAN_AKM_SUITE_FILS_SHA384                SUITE(0x000FAC, 15)
#define WLAN_AKM_SUITE_FT_FILS_SHA256                SUITE(0x000FAC, 16)
#define WLAN_AKM_SUITE_FT_FILS_SHA384                SUITE(0x000FAC, 17)
#define WLAN_AKM_SUITE_OWE                        SUITE(0x000FAC, 18)
#define WLAN_AKM_SUITE_FT_PSK_SHA384                SUITE(0x000FAC, 19)
#define WLAN_AKM_SUITE_PSK_SHA384                SUITE(0x000FAC, 20)

#define WLAN_AKM_SUITE_WFA_DPP                        SUITE(WLAN_OUI_WFA, 2)

#define WLAN_MAX_KEY_LEN                32

#define WLAN_PMK_NAME_LEN                16
#define WLAN_PMKID_LEN                        16
#define WLAN_PMK_LEN_EAP_LEAP                16
#define WLAN_PMK_LEN                        32
#define WLAN_PMK_LEN_SUITE_B_192        48

#define WLAN_OUI_WFA                        0x506f9a
#define WLAN_OUI_TYPE_WFA_P2P                9
#define WLAN_OUI_TYPE_WFA_NAN                0x13
#define WLAN_OUI_TYPE_WFA_DPP                0x1A
#define WLAN_OUI_MICROSOFT                0x0050f2
#define WLAN_OUI_TYPE_MICROSOFT_WPA        1
#define WLAN_OUI_TYPE_MICROSOFT_WMM        2
#define WLAN_OUI_TYPE_MICROSOFT_WPS        4
#define WLAN_OUI_TYPE_MICROSOFT_TPC        8

/*
 * WMM/802.11e Tspec Element
 */
#define IEEE80211_WMM_IE_TSPEC_TID_MASK                0x0F
#define IEEE80211_WMM_IE_TSPEC_TID_SHIFT        1

enum ieee80211_tspec_status_code {
        IEEE80211_TSPEC_STATUS_ADMISS_ACCEPTED = 0,
        IEEE80211_TSPEC_STATUS_ADDTS_INVAL_PARAMS = 0x1,
};

struct ieee80211_tspec_ie {
        u8 element_id;
        u8 len;
        u8 oui[3];
        u8 oui_type;
        u8 oui_subtype;
        u8 version;
        __le16 tsinfo;
        u8 tsinfo_resvd;
        __le16 nominal_msdu;
        __le16 max_msdu;
        __le32 min_service_int;
        __le32 max_service_int;
        __le32 inactivity_int;
        __le32 suspension_int;
        __le32 service_start_time;
        __le32 min_data_rate;
        __le32 mean_data_rate;
        __le32 peak_data_rate;
        __le32 max_burst_size;
        __le32 delay_bound;
        __le32 min_phy_rate;
        __le16 sba;
        __le16 medium_time;
} __packed;

/**
 * ieee80211_get_qos_ctl - get pointer to qos control bytes
 * @hdr: the frame
 * Return: a pointer to the QoS control field in the frame header
 *
 * The qos ctrl bytes come after the frame_control, duration, seq_num
 * and 3 or 4 addresses of length ETH_ALEN. Checks frame_control to choose
 * between struct ieee80211_qos_hdr_4addr and struct ieee80211_qos_hdr.
 */
static inline u8 *ieee80211_get_qos_ctl(struct ieee80211_hdr *hdr)
{
        union {
                struct ieee80211_qos_hdr        addr3;
                struct ieee80211_qos_hdr_4addr        addr4;
        } *qos;

        qos = (void *)hdr;
        if (ieee80211_has_a4(qos->addr3.frame_control))
                return (u8 *)&qos->addr4.qos_ctrl;
        else
                return (u8 *)&qos->addr3.qos_ctrl;
}

/**
 * ieee80211_get_tid - get qos TID
 * @hdr: the frame
 * Return: the TID from the QoS control field
 */
static inline u8 ieee80211_get_tid(struct ieee80211_hdr *hdr)
{
        u8 *qc = ieee80211_get_qos_ctl(hdr);

        return qc[0] & IEEE80211_QOS_CTL_TID_MASK;
}

/**
 * ieee80211_get_SA - get pointer to SA
 * @hdr: the frame
 * Return: a pointer to the source address (SA)
 *
 * Given an 802.11 frame, this function returns the offset
 * to the source address (SA). It does not verify that the
 * header is long enough to contain the address, and the
 * header must be long enough to contain the frame control
 * field.
 */
static inline u8 *ieee80211_get_SA(struct ieee80211_hdr *hdr)
{
        if (ieee80211_has_a4(hdr->frame_control))
                return hdr->addr4;
        if (ieee80211_has_fromds(hdr->frame_control))
                return hdr->addr3;
        return hdr->addr2;
}

/**
 * ieee80211_get_DA - get pointer to DA
 * @hdr: the frame
 * Return: a pointer to the destination address (DA)
 *
 * Given an 802.11 frame, this function returns the offset
 * to the destination address (DA). It does not verify that
 * the header is long enough to contain the address, and the
 * header must be long enough to contain the frame control
 * field.
 */
static inline u8 *ieee80211_get_DA(struct ieee80211_hdr *hdr)
{
        if (ieee80211_has_tods(hdr->frame_control))
                return hdr->addr3;
        else
                return hdr->addr1;
}

/**
 * ieee80211_is_bufferable_mmpdu - check if frame is bufferable MMPDU
 * @skb: the skb to check, starting with the 802.11 header
 * Return: whether or not the MMPDU is bufferable
 */
static inline bool ieee80211_is_bufferable_mmpdu(struct sk_buff *skb)
{
        struct ieee80211_mgmt *mgmt = (void *)skb->data;
        __le16 fc = mgmt->frame_control;

        /*
         * IEEE 802.11 REVme D2.0 definition of bufferable MMPDU;
         * note that this ignores the IBSS special case.
         */
        if (!ieee80211_is_mgmt(fc))
                return false;

        if (ieee80211_is_disassoc(fc) || ieee80211_is_deauth(fc))
                return true;

        if (!ieee80211_is_action(fc))
                return false;

        if (skb->len < IEEE80211_MIN_ACTION_SIZE(action_code))
                return true;

        /* action frame - additionally check for non-bufferable FTM */

        if (mgmt->u.action.category != WLAN_CATEGORY_PUBLIC &&
            mgmt->u.action.category != WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION)
                return true;

        if (mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_REQUEST ||
            mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_RESPONSE)
                return false;

        return true;
}

/**
 * _ieee80211_is_robust_mgmt_frame - check if frame is a robust management frame
 * @hdr: the frame (buffer must include at least the first octet of payload)
 * Return: whether or not the frame is a robust management frame
 */
static inline bool _ieee80211_is_robust_mgmt_frame(struct ieee80211_hdr *hdr)
{
        if (ieee80211_is_disassoc(hdr->frame_control) ||
            ieee80211_is_deauth(hdr->frame_control))
                return true;

        if (ieee80211_is_action(hdr->frame_control)) {
                u8 *category;

                /*
                 * Action frames, excluding Public Action frames, are Robust
                 * Management Frames. However, if we are looking at a Protected
                 * frame, skip the check since the data may be encrypted and
                 * the frame has already been found to be a Robust Management
                 * Frame (by the other end).
                 */
                if (ieee80211_has_protected(hdr->frame_control))
                        return true;
                category = ((u8 *) hdr) + 24;
                return *category != WLAN_CATEGORY_PUBLIC &&
                        *category != WLAN_CATEGORY_HT &&
                        *category != WLAN_CATEGORY_WNM_UNPROTECTED &&
                        *category != WLAN_CATEGORY_SELF_PROTECTED &&
                        *category != WLAN_CATEGORY_UNPROT_DMG &&
                        *category != WLAN_CATEGORY_VHT &&
                        *category != WLAN_CATEGORY_S1G &&
                        *category != WLAN_CATEGORY_VENDOR_SPECIFIC;
        }

        return false;
}

/**
 * ieee80211_is_robust_mgmt_frame - check if skb contains a robust mgmt frame
 * @skb: the skb containing the frame, length will be checked
 * Return: whether or not the frame is a robust management frame
 */
static inline bool ieee80211_is_robust_mgmt_frame(struct sk_buff *skb)
{
        if (skb->len < IEEE80211_MIN_ACTION_SIZE(category))
                return false;
        return _ieee80211_is_robust_mgmt_frame((void *)skb->data);
}

/**
 * ieee80211_is_public_action - check if frame is a public action frame
 * @hdr: the frame
 * @len: length of the frame
 * Return: whether or not the frame is a public action frame
 */
static inline bool ieee80211_is_public_action(struct ieee80211_hdr *hdr,
                                              size_t len)
{
        struct ieee80211_mgmt *mgmt = (void *)hdr;

        if (len < IEEE80211_MIN_ACTION_SIZE(category))
                return false;
        if (!ieee80211_is_action(hdr->frame_control))
                return false;
        return mgmt->u.action.category == WLAN_CATEGORY_PUBLIC;
}

/**
 * ieee80211_is_protected_dual_of_public_action - check if skb contains a
 * protected dual of public action management frame
 * @skb: the skb containing the frame, length will be checked
 *
 * Return: true if the skb contains a protected dual of public action
 * management frame, false otherwise.
 */
static inline bool
ieee80211_is_protected_dual_of_public_action(struct sk_buff *skb)
{
        struct ieee80211_mgmt *mgmt = (void *)skb->data;
        u8 action;

        if (!ieee80211_is_public_action((void *)skb->data, skb->len) ||
            skb->len < IEEE80211_MIN_ACTION_SIZE(action_code))
                return false;

        action = mgmt->u.action.action_code;

        return action != WLAN_PUB_ACTION_20_40_BSS_COEX &&
                action != WLAN_PUB_ACTION_DSE_REG_LOC_ANN &&
                action != WLAN_PUB_ACTION_MSMT_PILOT &&
                action != WLAN_PUB_ACTION_TDLS_DISCOVER_RES &&
                action != WLAN_PUB_ACTION_LOC_TRACK_NOTI &&
                action != WLAN_PUB_ACTION_FTM_REQUEST &&
                action != WLAN_PUB_ACTION_FTM_RESPONSE &&
                action != WLAN_PUB_ACTION_FILS_DISCOVERY &&
                action != WLAN_PUB_ACTION_VENDOR_SPECIFIC;
}

/**
 * _ieee80211_is_group_privacy_action - check if frame is a group addressed
 *        privacy action frame
 * @hdr: the frame
 * Return: whether or not the frame is a group addressed privacy action frame
 */
static inline bool _ieee80211_is_group_privacy_action(struct ieee80211_hdr *hdr)
{
        struct ieee80211_mgmt *mgmt = (void *)hdr;

        if (!ieee80211_is_action(hdr->frame_control) ||
            !is_multicast_ether_addr(hdr->addr1))
                return false;

        return mgmt->u.action.category == WLAN_CATEGORY_MESH_ACTION ||
               mgmt->u.action.category == WLAN_CATEGORY_MULTIHOP_ACTION;
}

/**
 * ieee80211_is_group_privacy_action - check if frame is a group addressed
 *        privacy action frame
 * @skb: the skb containing the frame, length will be checked
 * Return: whether or not the frame is a group addressed privacy action frame
 */
static inline bool ieee80211_is_group_privacy_action(struct sk_buff *skb)
{
        if (skb->len < IEEE80211_MIN_ACTION_SIZE(category))
                return false;
        return _ieee80211_is_group_privacy_action((void *)skb->data);
}

/**
 * ieee80211_tu_to_usec - convert time units (TU) to microseconds
 * @tu: the TUs
 * Return: the time value converted to microseconds
 */
static inline unsigned long ieee80211_tu_to_usec(unsigned long tu)
{
        return 1024 * tu;
}

static inline bool __ieee80211_check_tim(const struct ieee80211_tim_ie *tim,
                                         u8 tim_len, u16 aid)
{
        u8 mask;
        u8 index, indexn1, indexn2;

        if (unlikely(!tim || tim_len < sizeof(*tim)))
                return false;

        aid &= 0x3fff;
        index = aid / 8;
        mask  = 1 << (aid & 7);

        indexn1 = tim->bitmap_ctrl & 0xfe;
        indexn2 = tim_len + indexn1 - 4;

        if (index < indexn1 || index > indexn2)
                return false;

        index -= indexn1;

        return !!(tim->virtual_map[index] & mask);
}

/**
 * ieee80211_get_tdls_action - get TDLS action code
 * @skb: the skb containing the frame, length will not be checked
 * Return: the TDLS action code, or -1 if it's not an encapsulated TDLS action
 *        frame
 *
 * This function assumes the frame is a data frame, and that the network header
 * is in the correct place.
 */
static inline int ieee80211_get_tdls_action(struct sk_buff *skb)
{
        if (!skb_is_nonlinear(skb) &&
            skb->len > (skb_network_offset(skb) + 2)) {
                /* Point to where the indication of TDLS should start */
                const u8 *tdls_data = skb_network_header(skb) - 2;

                if (get_unaligned_be16(tdls_data) == ETH_P_TDLS &&
                    tdls_data[2] == WLAN_TDLS_SNAP_RFTYPE &&
                    tdls_data[3] == WLAN_CATEGORY_TDLS)
                        return tdls_data[4];
        }

        return -1;
}

/* convert time units */
#define TU_TO_JIFFIES(x)        (usecs_to_jiffies((x) * 1024))
#define TU_TO_EXP_TIME(x)        (jiffies + TU_TO_JIFFIES(x))

/* convert frequencies */
#define MHZ_TO_KHZ(freq) ((freq) * 1000)
#define KHZ_TO_MHZ(freq) ((freq) / 1000)
#define PR_KHZ(f) KHZ_TO_MHZ(f), f % 1000
#define KHZ_F "%d.%03d"

/* convert powers */
#define DBI_TO_MBI(gain) ((gain) * 100)
#define MBI_TO_DBI(gain) ((gain) / 100)
#define DBM_TO_MBM(gain) ((gain) * 100)
#define MBM_TO_DBM(gain) ((gain) / 100)

/**
 * ieee80211_action_contains_tpc - checks if the frame contains TPC element
 * @skb: the skb containing the frame, length will be checked
 * Return: %true if the frame contains a TPC element, %false otherwise
 *
 * This function checks if it's either TPC report action frame or Link
 * Measurement report action frame as defined in IEEE Std. 802.11-2012 8.5.2.5
 * and 8.5.7.5 accordingly.
 */
static inline bool ieee80211_action_contains_tpc(struct sk_buff *skb)
{
        struct ieee80211_mgmt *mgmt = (void *)skb->data;

        if (!ieee80211_is_action(mgmt->frame_control))
                return false;

        if (skb->len < IEEE80211_MIN_ACTION_SIZE(tpc_report))
                return false;

        /*
         * TPC report - check that:
         * category = 0 (Spectrum Management) or 5 (Radio Measurement)
         * spectrum management action = 3 (TPC/Link Measurement report)
         * TPC report EID = 35
         * TPC report element length = 2
         *
         * The spectrum management's tpc_report struct is used here both for
         * parsing tpc_report and radio measurement's link measurement report
         * frame, since the relevant part is identical in both frames.
         */
        if (mgmt->u.action.category != WLAN_CATEGORY_SPECTRUM_MGMT &&
            mgmt->u.action.category != WLAN_CATEGORY_RADIO_MEASUREMENT)
                return false;

        /* both spectrum mgmt and link measurement have same action code */
        if (mgmt->u.action.action_code != WLAN_ACTION_SPCT_TPC_RPRT)
                return false;

        if (mgmt->u.action.tpc_report.tpc_elem_id != WLAN_EID_TPC_REPORT ||
            mgmt->u.action.tpc_report.tpc_elem_length !=
            sizeof(struct ieee80211_tpc_report_ie))
                return false;

        return true;
}

/**
 * ieee80211_is_timing_measurement - check if frame is timing measurement response
 * @skb: the SKB to check
 * Return: whether or not the frame is a valid timing measurement response
 */
static inline bool ieee80211_is_timing_measurement(struct sk_buff *skb)
{
        struct ieee80211_mgmt *mgmt = (void *)skb->data;

        if (skb->len < IEEE80211_MIN_ACTION_SIZE(wnm_timing_msr))
                return false;

        if (!ieee80211_is_action(mgmt->frame_control))
                return false;

        if (mgmt->u.action.category == WLAN_CATEGORY_WNM_UNPROTECTED &&
            mgmt->u.action.action_code ==
                        WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE)
                return true;

        return false;
}

/**
 * ieee80211_is_ftm - check if frame is FTM response
 * @skb: the SKB to check
 * Return: whether or not the frame is a valid FTM response action frame
 */
static inline bool ieee80211_is_ftm(struct sk_buff *skb)
{
        struct ieee80211_mgmt *mgmt = (void *)skb->data;

        if (skb->len < IEEE80211_MIN_ACTION_SIZE(ftm))
                return false;

        if (!ieee80211_is_public_action((void *)mgmt, skb->len))
                return false;

        return mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_RESPONSE;
}

struct element {
        u8 id;
        u8 datalen;
        u8 data[];
} __packed;

/* element iteration helpers */
#define for_each_element(_elem, _data, _datalen)                        \
        for (_elem = (const struct element *)(_data);                        \
             (const u8 *)(_data) + (_datalen) - (const u8 *)_elem >=        \
                (int)sizeof(*_elem) &&                                        \
             (const u8 *)(_data) + (_datalen) - (const u8 *)_elem >=        \
                (int)sizeof(*_elem) + _elem->datalen;                        \
             _elem = (const struct element *)(_elem->data + _elem->datalen))

#define for_each_element_id(element, _id, data, datalen)                \
        for_each_element(element, data, datalen)                        \
                if (element->id == (_id))

#define for_each_element_extid(element, extid, _data, _datalen)                \
        for_each_element(element, _data, _datalen)                        \
                if (element->id == WLAN_EID_EXTENSION &&                \
                    element->datalen > 0 &&                                \
                    element->data[0] == (extid))

#define for_each_subelement(sub, element)                                \
        for_each_element(sub, (element)->data, (element)->datalen)

#define for_each_subelement_id(sub, id, element)                        \
        for_each_element_id(sub, id, (element)->data, (element)->datalen)

#define for_each_subelement_extid(sub, extid, element)                        \
        for_each_element_extid(sub, extid, (element)->data, (element)->datalen)

/**
 * for_each_element_completed - determine if element parsing consumed all data
 * @element: element pointer after for_each_element() or friends
 * @data: same data pointer as passed to for_each_element() or friends
 * @datalen: same data length as passed to for_each_element() or friends
 * Return: %true if all elements were iterated, %false otherwise; see notes
 *
 * This function returns %true if all the data was parsed or considered
 * while walking the elements. Only use this if your for_each_element()
 * loop cannot be broken out of, otherwise it always returns %false.
 *
 * If some data was malformed, this returns %false since the last parsed
 * element will not fill the whole remaining data.
 */
static inline bool for_each_element_completed(const struct element *element,
                                              const void *data, size_t datalen)
{
        return (const u8 *)element == (const u8 *)data + datalen;
}

/*
 * RSNX Capabilities:
 * bits 0-3: Field length (n-1)
 */
#define WLAN_RSNX_CAPA_PROTECTED_TWT BIT(4)
#define WLAN_RSNX_CAPA_SAE_H2E BIT(5)

/* EBPCC = Enhanced BSS Parameter Change Count */
#define IEEE80211_ENH_CRIT_UPD_EBPCC                0x0F
#define IEEE80211_ENH_CRIT_UPD_TYPE                0x70
#define IEEE80211_ENH_CRIT_UPD_TYPE_NO_UHR        0
#define IEEE80211_ENH_CRIT_UPD_TYPE_UHR                1
#define IEEE80211_ENH_CRIT_UPD_ALL                0x80

/**
 * struct ieee80211_enh_crit_upd - enhanced critical update (UHR)
 * @v: value of the enhanced critical update data,
 *        see %IEEE80211_ENH_CRIT_UPD_* to parse the bits
 */
struct ieee80211_enh_crit_upd {
        u8 v;
} __packed;

/*
 * reduced neighbor report, based on Draft P802.11ax_D6.1,
 * section 9.4.2.170 and accepted contributions.
 */
#define IEEE80211_AP_INFO_TBTT_HDR_TYPE                                0x03
#define IEEE80211_AP_INFO_TBTT_HDR_FILTERED                        0x04
#define IEEE80211_AP_INFO_TBTT_HDR_COLOC                        0x08
#define IEEE80211_AP_INFO_TBTT_HDR_COUNT                        0xF0
#define IEEE80211_TBTT_INFO_TYPE_TBTT                                0
#define IEEE80211_TBTT_INFO_TYPE_MLD                                1

#define IEEE80211_RNR_TBTT_PARAMS_OCT_RECOMMENDED                0x01
#define IEEE80211_RNR_TBTT_PARAMS_SAME_SSID                        0x02
#define IEEE80211_RNR_TBTT_PARAMS_MULTI_BSSID                        0x04
#define IEEE80211_RNR_TBTT_PARAMS_TRANSMITTED_BSSID                0x08
#define IEEE80211_RNR_TBTT_PARAMS_COLOC_ESS                        0x10
#define IEEE80211_RNR_TBTT_PARAMS_PROBE_ACTIVE                        0x20
#define IEEE80211_RNR_TBTT_PARAMS_COLOC_AP                        0x40
#define IEEE80211_RNR_TBTT_PARAMS_SAME_SMD                        0x80

#define IEEE80211_RNR_TBTT_PARAMS_PSD_NO_LIMIT                        127
#define IEEE80211_RNR_TBTT_PARAMS_PSD_RESERVED                        -128

struct ieee80211_neighbor_ap_info {
        u8 tbtt_info_hdr;
        u8 tbtt_info_len;
        u8 op_class;
        u8 channel;
} __packed;

enum ieee80211_range_params_max_total_ltf {
        IEEE80211_RANGE_PARAMS_MAX_TOTAL_LTF_4 = 0,
        IEEE80211_RANGE_PARAMS_MAX_TOTAL_LTF_8,
        IEEE80211_RANGE_PARAMS_MAX_TOTAL_LTF_16,
        IEEE80211_RANGE_PARAMS_MAX_TOTAL_LTF_UNSPECIFIED,
};

/*
 * reduced neighbor report, based on Draft P802.11be_D3.0,
 * section 9.4.2.170.2.
 */
struct ieee80211_rnr_mld_params {
        u8 mld_id;
        __le16 params;
} __packed;

#define IEEE80211_RNR_MLD_PARAMS_LINK_ID                        0x000F
#define IEEE80211_RNR_MLD_PARAMS_BSS_CHANGE_COUNT                0x0FF0
#define IEEE80211_RNR_MLD_PARAMS_UPDATES_INCLUDED                0x1000
#define IEEE80211_RNR_MLD_PARAMS_DISABLED_LINK                        0x2000

/* Format of the TBTT information element if it has 7, 8 or 9 bytes */
struct ieee80211_tbtt_info_7_8_9 {
        u8 tbtt_offset;
        u8 bssid[ETH_ALEN];

        /* The following element is optional, structure may not grow */
        u8 bss_params;
        s8 psd_20;
} __packed;

/* Format of the TBTT information element if it has >= 11 bytes */
struct ieee80211_tbtt_info_ge_11 {
        u8 tbtt_offset;
        u8 bssid[ETH_ALEN];
        __le32 short_ssid;

        /* The following elements are optional, structure may grow */
        u8 bss_params;
        s8 psd_20;
        struct ieee80211_rnr_mld_params mld_params;
        struct ieee80211_enh_crit_upd enh_crit_upd;
} __packed;

#include "ieee80211-ht.h"
#include "ieee80211-vht.h"
#include "ieee80211-he.h"
#include "ieee80211-eht.h"
#include "ieee80211-uhr.h"
#include "ieee80211-mesh.h"
#include "ieee80211-s1g.h"
#include "ieee80211-p2p.h"
#include "ieee80211-nan.h"

/**
 * ieee80211_check_tim - check if AID bit is set in TIM
 * @tim: the TIM IE
 * @tim_len: length of the TIM IE
 * @aid: the AID to look for
 * @s1g: whether the TIM is from an S1G PPDU
 * Return: whether or not traffic is indicated in the TIM for the given AID
 */
static inline bool ieee80211_check_tim(const struct ieee80211_tim_ie *tim,
                                       u8 tim_len, u16 aid, bool s1g)
{
        return s1g ? ieee80211_s1g_check_tim(tim, tim_len, aid) :
                     __ieee80211_check_tim(tim, tim_len, aid);
}

#endif /* LINUX_IEEE80211_H */



















































   32 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* X.509 certificate parser internal definitions
 *
 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#include <linux/cleanup.h>
#include <linux/time.h>
#include <crypto/public_key.h>
#include <keys/asymmetric-type.h>
#include <crypto/sha2.h>

struct x509_certificate {
        struct x509_certificate *next;
        struct x509_certificate *signer;        /* Certificate that signed this one */
        struct public_key *pub;                        /* Public key details */
        struct public_key_signature *sig;        /* Signature parameters */
        u8                sha256[SHA256_DIGEST_SIZE]; /* Hash for blacklist purposes */
        char                *issuer;                /* Name of certificate issuer */
        char                *subject;                /* Name of certificate subject */
        struct asymmetric_key_id *id;                /* Issuer + Serial number */
        struct asymmetric_key_id *skid;                /* Subject + subjectKeyId (optional) */
        time64_t        valid_from;
        time64_t        valid_to;
        const void        *tbs;                        /* Signed data */
        unsigned        tbs_size;                /* Size of signed data */
        unsigned        raw_sig_size;                /* Size of signature */
        const void        *raw_sig;                /* Signature data */
        const void        *raw_serial;                /* Raw serial number in ASN.1 */
        unsigned        raw_serial_size;
        unsigned        raw_issuer_size;
        const void        *raw_issuer;                /* Raw issuer name in ASN.1 */
        const void        *raw_subject;                /* Raw subject name in ASN.1 */
        unsigned        raw_subject_size;
        unsigned        raw_skid_size;
        const void        *raw_skid;                /* Raw subjectKeyId in ASN.1 */
        unsigned        index;
        bool                seen;                        /* Infinite recursion prevention */
        bool                verified;
        bool                self_signed;                /* T if self-signed (check unsupported_sig too) */
        bool                unsupported_sig;        /* T if signature uses unsupported crypto */
        bool                blacklisted;
};

/*
 * x509_cert_parser.c
 */
extern void x509_free_certificate(struct x509_certificate *cert);
DEFINE_FREE(x509_free_certificate, struct x509_certificate *,
            if (!IS_ERR(_T)) x509_free_certificate(_T))
extern struct x509_certificate *x509_cert_parse(const void *data, size_t datalen);
extern int x509_decode_time(time64_t *_t,  size_t hdrlen,
                            unsigned char tag,
                            const unsigned char *value, size_t vlen);

/*
 * x509_public_key.c
 */
extern int x509_get_sig_params(struct x509_certificate *cert);
extern int x509_check_for_self_signed(struct x509_certificate *cert);















































































































































































































































































































































































    8 











    8 
























































































































































































































































































































































































































































































































































































































































































































































































































































































































   12 































































































   12 





   12 





































   12 


   10 









   12 












































   12 


   12 






   10 


















    8 

























   12 


















   12 









   12 









   12 

















   12 




   12 



















   12 







   12 








   12 
    8 
    8 








   12 
   12 
   12 





















   10 
   10 






   12 







































   12 

























































   12 







   12 



   10 


   10 















   10 










   12 














































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/ext4/namei.c
 *
 * Copyright (C) 1992, 1993, 1994, 1995
 * Remy Card (card@masi.ibp.fr)
 * Laboratoire MASI - Institut Blaise Pascal
 * Universite Pierre et Marie Curie (Paris VI)
 *
 *  from
 *
 *  linux/fs/minix/namei.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 *  Big-endian to little-endian byte-swapping/bitmaps by
 *        David S. Miller (davem@caip.rutgers.edu), 1995
 *  Directory entry file type support and forward compatibility hooks
 *        for B-tree directories by Theodore Ts'o (tytso@mit.edu), 1998
 *  Hash Tree Directory indexing (c)
 *        Daniel Phillips, 2001
 *  Hash Tree Directory indexing porting
 *        Christopher Li, 2002
 *  Hash Tree Directory indexing cleanup
 *        Theodore Ts'o, 2002
 */

#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/time.h>
#include <linux/fcntl.h>
#include <linux/stat.h>
#include <linux/string.h>
#include <linux/quotaops.h>
#include <linux/buffer_head.h>
#include <linux/bio.h>
#include <linux/iversion.h>
#include <linux/unicode.h>
#include "ext4.h"
#include "ext4_jbd2.h"

#include "xattr.h"
#include "acl.h"

#include <trace/events/ext4.h>
/*
 * define how far ahead to read directories while searching them.
 */
#define NAMEI_RA_CHUNKS  2
#define NAMEI_RA_BLOCKS  4
#define NAMEI_RA_SIZE             (NAMEI_RA_CHUNKS * NAMEI_RA_BLOCKS)

static struct buffer_head *ext4_append(handle_t *handle,
                                        struct inode *inode,
                                        ext4_lblk_t *block)
{
        struct ext4_map_blocks map;
        struct buffer_head *bh;
        int err;

        if (unlikely(EXT4_SB(inode->i_sb)->s_max_dir_size_kb &&
                     ((inode->i_size >> 10) >=
                      EXT4_SB(inode->i_sb)->s_max_dir_size_kb)))
                return ERR_PTR(-ENOSPC);

        *block = inode->i_size >> inode->i_sb->s_blocksize_bits;
        map.m_lblk = *block;
        map.m_len = 1;

        /*
         * We're appending new directory block. Make sure the block is not
         * allocated yet, otherwise we will end up corrupting the
         * directory.
         */
        err = ext4_map_blocks(NULL, inode, &map, 0);
        if (err < 0)
                return ERR_PTR(err);
        if (err) {
                EXT4_ERROR_INODE(inode, "Logical block already allocated");
                return ERR_PTR(-EFSCORRUPTED);
        }

        bh = ext4_bread(handle, inode, *block, EXT4_GET_BLOCKS_CREATE);
        if (IS_ERR(bh))
                return bh;
        inode->i_size += inode->i_sb->s_blocksize;
        EXT4_I(inode)->i_disksize = inode->i_size;
        err = ext4_mark_inode_dirty(handle, inode);
        if (err)
                goto out;
        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, inode->i_sb, bh,
                                            EXT4_JTR_NONE);
        if (err)
                goto out;
        return bh;

out:
        brelse(bh);
        ext4_std_error(inode->i_sb, err);
        return ERR_PTR(err);
}

static int ext4_dx_csum_verify(struct inode *inode,
                               struct ext4_dir_entry *dirent);

/*
 * Hints to ext4_read_dirblock regarding whether we expect a directory
 * block being read to be an index block, or a block containing
 * directory entries (and if the latter, whether it was found via a
 * logical block in an htree index block).  This is used to control
 * what sort of sanity checkinig ext4_read_dirblock() will do on the
 * directory block read from the storage device.  EITHER will means
 * the caller doesn't know what kind of directory block will be read,
 * so no specific verification will be done.
 */
typedef enum {
        EITHER, INDEX, DIRENT, DIRENT_HTREE
} dirblock_type_t;

#define ext4_read_dirblock(inode, block, type) \
        __ext4_read_dirblock((inode), (block), (type), __func__, __LINE__)

static struct buffer_head *__ext4_read_dirblock(struct inode *inode,
                                                ext4_lblk_t block,
                                                dirblock_type_t type,
                                                const char *func,
                                                unsigned int line)
{
        struct buffer_head *bh;
        struct ext4_dir_entry *dirent;
        int is_dx_block = 0;

        if (block >= inode->i_size >> inode->i_blkbits) {
                ext4_error_inode(inode, func, line, block,
                       "Attempting to read directory block (%u) that is past i_size (%llu)",
                       block, inode->i_size);
                return ERR_PTR(-EFSCORRUPTED);
        }

        if (ext4_simulate_fail(inode->i_sb, EXT4_SIM_DIRBLOCK_EIO))
                bh = ERR_PTR(-EIO);
        else
                bh = ext4_bread(NULL, inode, block, 0);
        if (IS_ERR(bh)) {
                __ext4_warning(inode->i_sb, func, line,
                               "inode #%llu: lblock %lu: comm %s: "
                               "error %ld reading directory block",
                               inode->i_ino, (unsigned long)block,
                               current->comm, PTR_ERR(bh));

                return bh;
        }
        /* The first directory block must not be a hole. */
        if (!bh && (type == INDEX || type == DIRENT_HTREE || block == 0)) {
                ext4_error_inode(inode, func, line, block,
                                 "Directory hole found for htree %s block %u",
                                 (type == INDEX) ? "index" : "leaf", block);
                return ERR_PTR(-EFSCORRUPTED);
        }
        if (!bh)
                return NULL;
        dirent = (struct ext4_dir_entry *) bh->b_data;
        /* Determine whether or not we have an index block */
        if (is_dx(inode)) {
                if (block == 0)
                        is_dx_block = 1;
                else if (ext4_rec_len_from_disk(dirent->rec_len,
                                                inode->i_sb->s_blocksize) ==
                         inode->i_sb->s_blocksize)
                        is_dx_block = 1;
        }
        if (!is_dx_block && type == INDEX) {
                ext4_error_inode(inode, func, line, block,
                       "directory leaf block found instead of index block");
                brelse(bh);
                return ERR_PTR(-EFSCORRUPTED);
        }
        if (!ext4_has_feature_metadata_csum(inode->i_sb) ||
            buffer_verified(bh))
                return bh;

        /*
         * An empty leaf block can get mistaken for a index block; for
         * this reason, we can only check the index checksum when the
         * caller is sure it should be an index block.
         */
        if (is_dx_block && type == INDEX) {
                if (ext4_dx_csum_verify(inode, dirent) &&
                    !ext4_simulate_fail(inode->i_sb, EXT4_SIM_DIRBLOCK_CRC))
                        set_buffer_verified(bh);
                else {
                        ext4_error_inode_err(inode, func, line, block,
                                             EFSBADCRC,
                                             "Directory index failed checksum");
                        brelse(bh);
                        return ERR_PTR(-EFSBADCRC);
                }
        }
        if (!is_dx_block) {
                if (ext4_dirblock_csum_verify(inode, bh) &&
                    !ext4_simulate_fail(inode->i_sb, EXT4_SIM_DIRBLOCK_CRC))
                        set_buffer_verified(bh);
                else {
                        ext4_error_inode_err(inode, func, line, block,
                                             EFSBADCRC,
                                             "Directory block failed checksum");
                        brelse(bh);
                        return ERR_PTR(-EFSBADCRC);
                }
        }
        return bh;
}

#ifdef DX_DEBUG
#define dxtrace(command) command
#else
#define dxtrace(command)
#endif

struct fake_dirent
{
        __le32 inode;
        __le16 rec_len;
        u8 name_len;
        u8 file_type;
};

struct dx_countlimit
{
        __le16 limit;
        __le16 count;
};

struct dx_entry
{
        __le32 hash;
        __le32 block;
};

/*
 * dx_root_info is laid out so that if it should somehow get overlaid by a
 * dirent the two low bits of the hash version will be zero.  Therefore, the
 * hash version mod 4 should never be 0.  Sincerely, the paranoia department.
 */

struct dx_root
{
        struct fake_dirent dot;
        char dot_name[4];
        struct fake_dirent dotdot;
        char dotdot_name[4];
        struct dx_root_info
        {
                __le32 reserved_zero;
                u8 hash_version;
                u8 info_length; /* 8 */
                u8 indirect_levels;
                u8 unused_flags;
        }
        info;
        struct dx_entry        entries[];
};

struct dx_node
{
        struct fake_dirent fake;
        struct dx_entry        entries[];
};


struct dx_frame
{
        struct buffer_head *bh;
        struct dx_entry *entries;
        struct dx_entry *at;
};

struct dx_map_entry
{
        u32 hash;
        u16 offs;
        u16 size;
};

/*
 * This goes at the end of each htree block.
 */
struct dx_tail {
        u32 dt_reserved;
        __le32 dt_checksum;        /* crc32c(uuid+inum+dirblock) */
};

static struct buffer_head * ext4_dx_find_entry(struct inode *dir,
                struct ext4_filename *fname,
                struct ext4_dir_entry_2 **res_dir);
static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname,
                             struct inode *dir, struct inode *inode);

/* checksumming functions */
void ext4_initialize_dirent_tail(struct buffer_head *bh,
                                 unsigned int blocksize)
{
        struct ext4_dir_entry_tail *t = EXT4_DIRENT_TAIL(bh->b_data, blocksize);

        memset(t, 0, sizeof(struct ext4_dir_entry_tail));
        t->det_rec_len = ext4_rec_len_to_disk(
                        sizeof(struct ext4_dir_entry_tail), blocksize);
        t->det_reserved_ft = EXT4_FT_DIR_CSUM;
}

/* Walk through a dirent block to find a checksum "dirent" at the tail */
static struct ext4_dir_entry_tail *get_dirent_tail(struct inode *inode,
                                                   struct buffer_head *bh)
{
        struct ext4_dir_entry_tail *t;
        int blocksize = EXT4_BLOCK_SIZE(inode->i_sb);

#ifdef PARANOID
        struct ext4_dir_entry *d, *top;

        d = (struct ext4_dir_entry *)bh->b_data;
        top = (struct ext4_dir_entry *)(bh->b_data +
                (blocksize - sizeof(struct ext4_dir_entry_tail)));
        while (d < top && ext4_rec_len_from_disk(d->rec_len, blocksize))
                d = (struct ext4_dir_entry *)(((void *)d) +
                    ext4_rec_len_from_disk(d->rec_len, blocksize));

        if (d != top)
                return NULL;

        t = (struct ext4_dir_entry_tail *)d;
#else
        t = EXT4_DIRENT_TAIL(bh->b_data, EXT4_BLOCK_SIZE(inode->i_sb));
#endif

        if (t->det_reserved_zero1 ||
            (ext4_rec_len_from_disk(t->det_rec_len, blocksize) !=
             sizeof(struct ext4_dir_entry_tail)) ||
            t->det_reserved_zero2 ||
            t->det_reserved_ft != EXT4_FT_DIR_CSUM)
                return NULL;

        return t;
}

static __le32 ext4_dirblock_csum(struct inode *inode, void *dirent, int size)
{
        struct ext4_inode_info *ei = EXT4_I(inode);
        __u32 csum;

        csum = ext4_chksum(ei->i_csum_seed, (__u8 *)dirent, size);
        return cpu_to_le32(csum);
}

#define warn_no_space_for_csum(inode)                                        \
        __warn_no_space_for_csum((inode), __func__, __LINE__)

static void __warn_no_space_for_csum(struct inode *inode, const char *func,
                                     unsigned int line)
{
        __ext4_warning_inode(inode, func, line,
                "No space for directory leaf checksum. Please run e2fsck -D.");
}

int ext4_dirblock_csum_verify(struct inode *inode, struct buffer_head *bh)
{
        struct ext4_dir_entry_tail *t;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return 1;

        t = get_dirent_tail(inode, bh);
        if (!t) {
                warn_no_space_for_csum(inode);
                return 0;
        }

        if (t->det_checksum != ext4_dirblock_csum(inode, bh->b_data,
                                                  (char *)t - bh->b_data))
                return 0;

        return 1;
}

static void ext4_dirblock_csum_set(struct inode *inode,
                                 struct buffer_head *bh)
{
        struct ext4_dir_entry_tail *t;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return;

        t = get_dirent_tail(inode, bh);
        if (!t) {
                warn_no_space_for_csum(inode);
                return;
        }

        t->det_checksum = ext4_dirblock_csum(inode, bh->b_data,
                                             (char *)t - bh->b_data);
}

int ext4_handle_dirty_dirblock(handle_t *handle,
                               struct inode *inode,
                               struct buffer_head *bh)
{
        ext4_dirblock_csum_set(inode, bh);
        return ext4_handle_dirty_metadata(handle, inode, bh);
}

static struct dx_countlimit *get_dx_countlimit(struct inode *inode,
                                               struct ext4_dir_entry *dirent,
                                               int *offset)
{
        struct ext4_dir_entry *dp;
        struct dx_root_info *root;
        int count_offset;
        int blocksize = EXT4_BLOCK_SIZE(inode->i_sb);
        unsigned int rlen = ext4_rec_len_from_disk(dirent->rec_len, blocksize);

        if (rlen == blocksize)
                count_offset = 8;
        else if (rlen == 12) {
                dp = (struct ext4_dir_entry *)(((void *)dirent) + 12);
                if (ext4_rec_len_from_disk(dp->rec_len, blocksize) != blocksize - 12)
                        return NULL;
                root = (struct dx_root_info *)(((void *)dp + 12));
                if (root->reserved_zero ||
                    root->info_length != sizeof(struct dx_root_info))
                        return NULL;
                count_offset = 32;
        } else
                return NULL;

        if (offset)
                *offset = count_offset;
        return (struct dx_countlimit *)(((void *)dirent) + count_offset);
}

static __le32 ext4_dx_csum(struct inode *inode, struct ext4_dir_entry *dirent,
                           int count_offset, int count, struct dx_tail *t)
{
        struct ext4_inode_info *ei = EXT4_I(inode);
        __u32 csum;
        int size;
        __u32 dummy_csum = 0;
        int offset = offsetof(struct dx_tail, dt_checksum);

        size = count_offset + (count * sizeof(struct dx_entry));
        csum = ext4_chksum(ei->i_csum_seed, (__u8 *)dirent, size);
        csum = ext4_chksum(csum, (__u8 *)t, offset);
        csum = ext4_chksum(csum, (__u8 *)&dummy_csum, sizeof(dummy_csum));

        return cpu_to_le32(csum);
}

static int ext4_dx_csum_verify(struct inode *inode,
                               struct ext4_dir_entry *dirent)
{
        struct dx_countlimit *c;
        struct dx_tail *t;
        int count_offset, limit, count;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return 1;

        c = get_dx_countlimit(inode, dirent, &count_offset);
        if (!c) {
                EXT4_ERROR_INODE(inode, "dir seems corrupt?  Run e2fsck -D.");
                return 0;
        }
        limit = le16_to_cpu(c->limit);
        count = le16_to_cpu(c->count);
        if (count_offset + (limit * sizeof(struct dx_entry)) >
            EXT4_BLOCK_SIZE(inode->i_sb) - sizeof(struct dx_tail)) {
                warn_no_space_for_csum(inode);
                return 0;
        }
        t = (struct dx_tail *)(((struct dx_entry *)c) + limit);

        if (t->dt_checksum != ext4_dx_csum(inode, dirent, count_offset,
                                            count, t))
                return 0;
        return 1;
}

static void ext4_dx_csum_set(struct inode *inode, struct ext4_dir_entry *dirent)
{
        struct dx_countlimit *c;
        struct dx_tail *t;
        int count_offset, limit, count;

        if (!ext4_has_feature_metadata_csum(inode->i_sb))
                return;

        c = get_dx_countlimit(inode, dirent, &count_offset);
        if (!c) {
                EXT4_ERROR_INODE(inode, "dir seems corrupt?  Run e2fsck -D.");
                return;
        }
        limit = le16_to_cpu(c->limit);
        count = le16_to_cpu(c->count);
        if (count_offset + (limit * sizeof(struct dx_entry)) >
            EXT4_BLOCK_SIZE(inode->i_sb) - sizeof(struct dx_tail)) {
                warn_no_space_for_csum(inode);
                return;
        }
        t = (struct dx_tail *)(((struct dx_entry *)c) + limit);

        t->dt_checksum = ext4_dx_csum(inode, dirent, count_offset, count, t);
}

static inline int ext4_handle_dirty_dx_node(handle_t *handle,
                                            struct inode *inode,
                                            struct buffer_head *bh)
{
        ext4_dx_csum_set(inode, (struct ext4_dir_entry *)bh->b_data);
        return ext4_handle_dirty_metadata(handle, inode, bh);
}

/*
 * p is at least 6 bytes before the end of page
 */
static inline struct ext4_dir_entry_2 *
ext4_next_entry(struct ext4_dir_entry_2 *p, unsigned long blocksize)
{
        return (struct ext4_dir_entry_2 *)((char *)p +
                ext4_rec_len_from_disk(p->rec_len, blocksize));
}

/*
 * Future: use high four bits of block for coalesce-on-delete flags
 * Mask them off for now.
 */

static inline ext4_lblk_t dx_get_block(struct dx_entry *entry)
{
        return le32_to_cpu(entry->block) & 0x0fffffff;
}

static inline void dx_set_block(struct dx_entry *entry, ext4_lblk_t value)
{
        entry->block = cpu_to_le32(value);
}

static inline unsigned dx_get_hash(struct dx_entry *entry)
{
        return le32_to_cpu(entry->hash);
}

static inline void dx_set_hash(struct dx_entry *entry, unsigned value)
{
        entry->hash = cpu_to_le32(value);
}

static inline unsigned dx_get_count(struct dx_entry *entries)
{
        return le16_to_cpu(((struct dx_countlimit *) entries)->count);
}

static inline unsigned dx_get_limit(struct dx_entry *entries)
{
        return le16_to_cpu(((struct dx_countlimit *) entries)->limit);
}

static inline void dx_set_count(struct dx_entry *entries, unsigned value)
{
        ((struct dx_countlimit *) entries)->count = cpu_to_le16(value);
}

static inline void dx_set_limit(struct dx_entry *entries, unsigned value)
{
        ((struct dx_countlimit *) entries)->limit = cpu_to_le16(value);
}

static inline unsigned dx_root_limit(struct inode *dir, unsigned infosize)
{
        unsigned int entry_space = dir->i_sb->s_blocksize -
                        ext4_dir_rec_len(1, NULL) -
                        ext4_dir_rec_len(2, NULL) - infosize;

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                entry_space -= sizeof(struct dx_tail);
        return entry_space / sizeof(struct dx_entry);
}

static inline unsigned dx_node_limit(struct inode *dir)
{
        unsigned int entry_space = dir->i_sb->s_blocksize -
                        ext4_dir_rec_len(0, dir);

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                entry_space -= sizeof(struct dx_tail);
        return entry_space / sizeof(struct dx_entry);
}

/*
 * Debug
 */
#ifdef DX_DEBUG
static void dx_show_index(char * label, struct dx_entry *entries)
{
        int i, n = dx_get_count (entries);
        printk(KERN_DEBUG "%s index", label);
        for (i = 0; i < n; i++) {
                printk(KERN_CONT " %x->%lu",
                       i ? dx_get_hash(entries + i) : 0,
                       (unsigned long)dx_get_block(entries + i));
        }
        printk(KERN_CONT "\n");
}

struct stats
{
        unsigned names;
        unsigned space;
        unsigned bcount;
};

static struct stats dx_show_leaf(struct inode *dir,
                                struct dx_hash_info *hinfo,
                                struct ext4_dir_entry_2 *de,
                                int size, int show_names)
{
        unsigned names = 0, space = 0;
        char *base = (char *) de;
        struct dx_hash_info h = *hinfo;

        printk("names: ");
        while ((char *) de < base + size)
        {
                if (de->inode)
                {
                        if (show_names)
                        {
#ifdef CONFIG_FS_ENCRYPTION
                                int len;
                                char *name;
                                struct fscrypt_str fname_crypto_str =
                                        FSTR_INIT(NULL, 0);
                                int res = 0;

                                name  = de->name;
                                len = de->name_len;
                                if (!IS_ENCRYPTED(dir)) {
                                        /* Directory is not encrypted */
                                        (void) ext4fs_dirhash(dir, de->name,
                                                de->name_len, &h);
                                        printk("%.*s:(U)%x.%u ", len,
                                               name, h.hash,
                                               (unsigned) ((char *) de
                                                           - base));
                                } else {
                                        struct fscrypt_str de_name =
                                                FSTR_INIT(name, len);

                                        /* Directory is encrypted */
                                        res = fscrypt_fname_alloc_buffer(
                                                len, &fname_crypto_str);
                                        if (res)
                                                printk(KERN_WARNING "Error "
                                                        "allocating crypto "
                                                        "buffer--skipping "
                                                        "crypto\n");
                                        res = fscrypt_fname_disk_to_usr(dir,
                                                0, 0, &de_name,
                                                &fname_crypto_str);
                                        if (res) {
                                                printk(KERN_WARNING "Error "
                                                        "converting filename "
                                                        "from disk to usr"
                                                        "\n");
                                                name = "??";
                                                len = 2;
                                        } else {
                                                name = fname_crypto_str.name;
                                                len = fname_crypto_str.len;
                                        }
                                        if (IS_CASEFOLDED(dir))
                                                h.hash = EXT4_DIRENT_HASH(de);
                                        else
                                                (void) ext4fs_dirhash(dir,
                                                        de->name,
                                                        de->name_len, &h);
                                        printk("%.*s:(E)%x.%u ", len, name,
                                               h.hash, (unsigned) ((char *) de
                                                                   - base));
                                        fscrypt_fname_free_buffer(
                                                        &fname_crypto_str);
                                }
#else
                                int len = de->name_len;
                                char *name = de->name;
                                (void) ext4fs_dirhash(dir, de->name,
                                                      de->name_len, &h);
                                printk("%.*s:%x.%u ", len, name, h.hash,
                                       (unsigned) ((char *) de - base));
#endif
                        }
                        space += ext4_dir_rec_len(de->name_len, dir);
                        names++;
                }
                de = ext4_next_entry(de, size);
        }
        printk(KERN_CONT "(%i)\n", names);
        return (struct stats) { names, space, 1 };
}

struct stats dx_show_entries(struct dx_hash_info *hinfo, struct inode *dir,
                             struct dx_entry *entries, int levels)
{
        unsigned blocksize = dir->i_sb->s_blocksize;
        unsigned count = dx_get_count(entries), names = 0, space = 0, i;
        unsigned bcount = 0;
        struct buffer_head *bh;
        printk("%i indexed blocks...\n", count);
        for (i = 0; i < count; i++, entries++)
        {
                ext4_lblk_t block = dx_get_block(entries);
                ext4_lblk_t hash  = i ? dx_get_hash(entries): 0;
                u32 range = i < count - 1? (dx_get_hash(entries + 1) - hash): ~hash;
                struct stats stats;
                printk("%s%3u:%03u hash %8x/%8x ",levels?"":"   ", i, block, hash, range);
                bh = ext4_bread(NULL,dir, block, 0);
                if (IS_ERR_OR_NULL(bh))
                        continue;
                stats = levels?
                   dx_show_entries(hinfo, dir, ((struct dx_node *) bh->b_data)->entries, levels - 1):
                   dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *)
                        bh->b_data, blocksize, 0);
                names += stats.names;
                space += stats.space;
                bcount += stats.bcount;
                brelse(bh);
        }
        if (bcount)
                printk(KERN_DEBUG "%snames %u, fullness %u (%u%%)\n",
                       levels ? "" : "   ", names, space/bcount,
                       (space/bcount)*100/blocksize);
        return (struct stats) { names, space, bcount};
}

/*
 * Linear search cross check
 */
static inline void htree_rep_invariant_check(struct dx_entry *at,
                                             struct dx_entry *target,
                                             u32 hash, unsigned int n)
{
        while (n--) {
                dxtrace(printk(KERN_CONT ","));
                if (dx_get_hash(++at) > hash) {
                        at--;
                        break;
                }
        }
        ASSERT(at == target - 1);
}
#else /* DX_DEBUG */
static inline void htree_rep_invariant_check(struct dx_entry *at,
                                             struct dx_entry *target,
                                             u32 hash, unsigned int n)
{
}
#endif /* DX_DEBUG */

/*
 * Probe for a directory leaf block to search.
 *
 * dx_probe can return ERR_BAD_DX_DIR, which means there was a format
 * error in the directory index, and the caller should fall back to
 * searching the directory normally.  The callers of dx_probe **MUST**
 * check for this error code, and make sure it never gets reflected
 * back to userspace.
 */
static struct dx_frame *
dx_probe(struct ext4_filename *fname, struct inode *dir,
         struct dx_hash_info *hinfo, struct dx_frame *frame_in)
{
        unsigned count, indirect, level, i;
        struct dx_entry *at, *entries, *p, *q, *m;
        struct dx_root *root;
        struct dx_frame *frame = frame_in;
        struct dx_frame *ret_err = ERR_PTR(ERR_BAD_DX_DIR);
        u32 hash;
        ext4_lblk_t block;
        ext4_lblk_t blocks[EXT4_HTREE_LEVEL];

        memset(frame_in, 0, EXT4_HTREE_LEVEL * sizeof(frame_in[0]));
        frame->bh = ext4_read_dirblock(dir, 0, INDEX);
        if (IS_ERR(frame->bh))
                return (struct dx_frame *) frame->bh;

        root = (struct dx_root *) frame->bh->b_data;
        if (root->info.hash_version != DX_HASH_TEA &&
            root->info.hash_version != DX_HASH_HALF_MD4 &&
            root->info.hash_version != DX_HASH_LEGACY &&
            root->info.hash_version != DX_HASH_SIPHASH) {
                ext4_warning_inode(dir, "Unrecognised inode hash code %u",
                                   root->info.hash_version);
                goto fail;
        }
        if (ext4_hash_in_dirent(dir)) {
                if (root->info.hash_version != DX_HASH_SIPHASH) {
                        ext4_warning_inode(dir,
                                "Hash in dirent, but hash is not SIPHASH");
                        goto fail;
                }
        } else {
                if (root->info.hash_version == DX_HASH_SIPHASH) {
                        ext4_warning_inode(dir,
                                "Hash code is SIPHASH, but hash not in dirent");
                        goto fail;
                }
        }
        if (fname)
                hinfo = &fname->hinfo;
        hinfo->hash_version = root->info.hash_version;
        if (hinfo->hash_version <= DX_HASH_TEA)
                hinfo->hash_version += EXT4_SB(dir->i_sb)->s_hash_unsigned;
        hinfo->seed = EXT4_SB(dir->i_sb)->s_hash_seed;
        /* hash is already computed for encrypted casefolded directory */
        if (fname && fname_name(fname) &&
            !(IS_ENCRYPTED(dir) && IS_CASEFOLDED(dir))) {
                int ret = ext4fs_dirhash(dir, fname_name(fname),
                                         fname_len(fname), hinfo);
                if (ret < 0) {
                        ret_err = ERR_PTR(ret);
                        goto fail;
                }
        }
        hash = hinfo->hash;

        if (root->info.unused_flags & 1) {
                ext4_warning_inode(dir, "Unimplemented hash flags: %#06x",
                                   root->info.unused_flags);
                goto fail;
        }

        indirect = root->info.indirect_levels;
        if (indirect >= ext4_dir_htree_level(dir->i_sb)) {
                ext4_warning(dir->i_sb,
                             "Directory (ino: %llu) htree depth %#06x exceed"
                             "supported value", dir->i_ino,
                             ext4_dir_htree_level(dir->i_sb));
                if (ext4_dir_htree_level(dir->i_sb) < EXT4_HTREE_LEVEL) {
                        ext4_warning(dir->i_sb, "Enable large directory "
                                                "feature to access it");
                }
                goto fail;
        }

        entries = (struct dx_entry *)(((char *)&root->info) +
                                      root->info.info_length);

        if (dx_get_limit(entries) != dx_root_limit(dir,
                                                   root->info.info_length)) {
                ext4_warning_inode(dir, "dx entry: limit %u != root limit %u",
                                   dx_get_limit(entries),
                                   dx_root_limit(dir, root->info.info_length));
                goto fail;
        }

        dxtrace(printk("Look up %x", hash));
        level = 0;
        blocks[0] = 0;
        while (1) {
                count = dx_get_count(entries);
                if (!count || count > dx_get_limit(entries)) {
                        ext4_warning_inode(dir,
                                           "dx entry: count %u beyond limit %u",
                                           count, dx_get_limit(entries));
                        goto fail;
                }

                p = entries + 1;
                q = entries + count - 1;
                while (p <= q) {
                        m = p + (q - p) / 2;
                        dxtrace(printk(KERN_CONT "."));
                        if (dx_get_hash(m) > hash)
                                q = m - 1;
                        else
                                p = m + 1;
                }

                htree_rep_invariant_check(entries, p, hash, count - 1);

                at = p - 1;
                dxtrace(printk(KERN_CONT " %x->%u\n",
                               at == entries ? 0 : dx_get_hash(at),
                               dx_get_block(at)));
                frame->entries = entries;
                frame->at = at;

                block = dx_get_block(at);
                for (i = 0; i <= level; i++) {
                        if (blocks[i] == block) {
                                ext4_warning_inode(dir,
                                        "dx entry: tree cycle block %u points back to block %u",
                                        blocks[level], block);
                                goto fail;
                        }
                }
                if (++level > indirect)
                        return frame;
                blocks[level] = block;
                frame++;
                frame->bh = ext4_read_dirblock(dir, block, INDEX);
                if (IS_ERR(frame->bh)) {
                        ret_err = (struct dx_frame *) frame->bh;
                        frame->bh = NULL;
                        goto fail;
                }

                entries = ((struct dx_node *) frame->bh->b_data)->entries;

                if (dx_get_limit(entries) != dx_node_limit(dir)) {
                        ext4_warning_inode(dir,
                                "dx entry: limit %u != node limit %u",
                                dx_get_limit(entries), dx_node_limit(dir));
                        goto fail;
                }
        }
fail:
        while (frame >= frame_in) {
                brelse(frame->bh);
                frame--;
        }

        if (ret_err == ERR_PTR(ERR_BAD_DX_DIR))
                ext4_warning_inode(dir,
                        "Corrupt directory, running e2fsck is recommended");
        return ret_err;
}

static void dx_release(struct dx_frame *frames)
{
        struct dx_root_info *info;
        int i;
        unsigned int indirect_levels;

        if (frames[0].bh == NULL)
                return;

        info = &((struct dx_root *)frames[0].bh->b_data)->info;
        /* save local copy, "info" may be freed after brelse() */
        indirect_levels = info->indirect_levels;
        for (i = 0; i <= indirect_levels; i++) {
                if (frames[i].bh == NULL)
                        break;
                brelse(frames[i].bh);
                frames[i].bh = NULL;
        }
}

/*
 * This function increments the frame pointer to search the next leaf
 * block, and reads in the necessary intervening nodes if the search
 * should be necessary.  Whether or not the search is necessary is
 * controlled by the hash parameter.  If the hash value is even, then
 * the search is only continued if the next block starts with that
 * hash value.  This is used if we are searching for a specific file.
 *
 * If the hash value is HASH_NB_ALWAYS, then always go to the next block.
 *
 * This function returns 1 if the caller should continue to search,
 * or 0 if it should not.  If there is an error reading one of the
 * index blocks, it will a negative error code.
 *
 * If start_hash is non-null, it will be filled in with the starting
 * hash of the next page.
 */
static int ext4_htree_next_block(struct inode *dir, __u32 hash,
                                 struct dx_frame *frame,
                                 struct dx_frame *frames,
                                 __u32 *start_hash)
{
        struct dx_frame *p;
        struct buffer_head *bh;
        int num_frames = 0;
        __u32 bhash;

        p = frame;
        /*
         * Find the next leaf page by incrementing the frame pointer.
         * If we run out of entries in the interior node, loop around and
         * increment pointer in the parent node.  When we break out of
         * this loop, num_frames indicates the number of interior
         * nodes need to be read.
         */
        while (1) {
                if (++(p->at) < p->entries + dx_get_count(p->entries))
                        break;
                if (p == frames)
                        return 0;
                num_frames++;
                p--;
        }

        /*
         * If the hash is 1, then continue only if the next page has a
         * continuation hash of any value.  This is used for readdir
         * handling.  Otherwise, check to see if the hash matches the
         * desired continuation hash.  If it doesn't, return since
         * there's no point to read in the successive index pages.
         */
        bhash = dx_get_hash(p->at);
        if (start_hash)
                *start_hash = bhash;
        if ((hash & 1) == 0) {
                if ((bhash & ~1) != hash)
                        return 0;
        }
        /*
         * If the hash is HASH_NB_ALWAYS, we always go to the next
         * block so no check is necessary
         */
        while (num_frames--) {
                bh = ext4_read_dirblock(dir, dx_get_block(p->at), INDEX);
                if (IS_ERR(bh))
                        return PTR_ERR(bh);
                p++;
                brelse(p->bh);
                p->bh = bh;
                p->at = p->entries = ((struct dx_node *) bh->b_data)->entries;
        }
        return 1;
}


/*
 * This function fills a red-black tree with information from a
 * directory block.  It returns the number directory entries loaded
 * into the tree.  If there is an error it is returned in err.
 */
static int htree_dirblock_to_tree(struct file *dir_file,
                                  struct inode *dir, ext4_lblk_t block,
                                  struct dx_hash_info *hinfo,
                                  __u32 start_hash, __u32 start_minor_hash)
{
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de, *top;
        int err = 0, count = 0;
        struct fscrypt_str fname_crypto_str = FSTR_INIT(NULL, 0), tmp_str;
        int csum = ext4_has_feature_metadata_csum(dir->i_sb);

        dxtrace(printk(KERN_INFO "In htree dirblock_to_tree: block %lu\n",
                                                        (unsigned long)block));
        bh = ext4_read_dirblock(dir, block, DIRENT_HTREE);
        if (IS_ERR(bh))
                return PTR_ERR(bh);

        de = (struct ext4_dir_entry_2 *) bh->b_data;
        /* csum entries are not larger in the casefolded encrypted case */
        top = (struct ext4_dir_entry_2 *) ((char *) de +
                                           dir->i_sb->s_blocksize -
                                           ext4_dir_rec_len(0,
                                                           csum ? NULL : dir));
        /* Check if the directory is encrypted */
        if (IS_ENCRYPTED(dir)) {
                err = fscrypt_prepare_readdir(dir);
                if (err < 0) {
                        brelse(bh);
                        return err;
                }
                err = fscrypt_fname_alloc_buffer(EXT4_NAME_LEN,
                                                 &fname_crypto_str);
                if (err < 0) {
                        brelse(bh);
                        return err;
                }
        }

        for (; de < top; de = ext4_next_entry(de, dir->i_sb->s_blocksize)) {
                if (ext4_check_dir_entry(dir, NULL, de, bh,
                                bh->b_data, bh->b_size,
                                EXT4_LBLK_TO_B(dir, block)
                                         + ((char *)de - bh->b_data))) {
                        /* silently ignore the rest of the block */
                        break;
                }
                if (ext4_hash_in_dirent(dir)) {
                        if (de->name_len && de->inode) {
                                hinfo->hash = EXT4_DIRENT_HASH(de);
                                hinfo->minor_hash = EXT4_DIRENT_MINOR_HASH(de);
                        } else {
                                hinfo->hash = 0;
                                hinfo->minor_hash = 0;
                        }
                } else {
                        err = ext4fs_dirhash(dir, de->name,
                                             de->name_len, hinfo);
                        if (err < 0) {
                                count = err;
                                goto errout;
                        }
                }
                if ((hinfo->hash < start_hash) ||
                    ((hinfo->hash == start_hash) &&
                     (hinfo->minor_hash < start_minor_hash)))
                        continue;
                if (de->inode == 0)
                        continue;
                if (!IS_ENCRYPTED(dir)) {
                        tmp_str.name = de->name;
                        tmp_str.len = de->name_len;
                        err = ext4_htree_store_dirent(dir_file,
                                   hinfo->hash, hinfo->minor_hash, de,
                                   &tmp_str);
                } else {
                        int save_len = fname_crypto_str.len;
                        struct fscrypt_str de_name = FSTR_INIT(de->name,
                                                                de->name_len);

                        /* Directory is encrypted */
                        err = fscrypt_fname_disk_to_usr(dir, hinfo->hash,
                                        hinfo->minor_hash, &de_name,
                                        &fname_crypto_str);
                        if (err) {
                                count = err;
                                goto errout;
                        }
                        err = ext4_htree_store_dirent(dir_file,
                                   hinfo->hash, hinfo->minor_hash, de,
                                        &fname_crypto_str);
                        fname_crypto_str.len = save_len;
                }
                if (err != 0) {
                        count = err;
                        goto errout;
                }
                count++;
        }
errout:
        brelse(bh);
        fscrypt_fname_free_buffer(&fname_crypto_str);
        return count;
}


/*
 * This function fills a red-black tree with information from a
 * directory.  We start scanning the directory in hash order, starting
 * at start_hash and start_minor_hash.
 *
 * This function returns the number of entries inserted into the tree,
 * or a negative error code.
 */
int ext4_htree_fill_tree(struct file *dir_file, __u32 start_hash,
                         __u32 start_minor_hash, __u32 *next_hash)
{
        struct dx_hash_info hinfo;
        struct ext4_dir_entry_2 *de;
        struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
        struct inode *dir;
        ext4_lblk_t block;
        int count = 0;
        int ret, err;
        __u32 hashval;
        struct fscrypt_str tmp_str;

        dxtrace(printk(KERN_DEBUG "In htree_fill_tree, start hash: %x:%x\n",
                       start_hash, start_minor_hash));
        dir = file_inode(dir_file);
        if (!(ext4_test_inode_flag(dir, EXT4_INODE_INDEX))) {
                if (ext4_hash_in_dirent(dir))
                        hinfo.hash_version = DX_HASH_SIPHASH;
                else
                        hinfo.hash_version =
                                        EXT4_SB(dir->i_sb)->s_def_hash_version;
                if (hinfo.hash_version <= DX_HASH_TEA)
                        hinfo.hash_version +=
                                EXT4_SB(dir->i_sb)->s_hash_unsigned;
                hinfo.seed = EXT4_SB(dir->i_sb)->s_hash_seed;
                if (ext4_has_inline_data(dir)) {
                        int has_inline_data = 1;
                        count = ext4_inlinedir_to_tree(dir_file, dir, 0,
                                                       &hinfo, start_hash,
                                                       start_minor_hash,
                                                       &has_inline_data);
                        if (has_inline_data) {
                                *next_hash = ~0;
                                return count;
                        }
                }
                count = htree_dirblock_to_tree(dir_file, dir, 0, &hinfo,
                                               start_hash, start_minor_hash);
                *next_hash = ~0;
                return count;
        }
        hinfo.hash = start_hash;
        hinfo.minor_hash = 0;
        frame = dx_probe(NULL, dir, &hinfo, frames);
        if (IS_ERR(frame))
                return PTR_ERR(frame);

        /* Add '.' and '..' from the htree header */
        if (!start_hash && !start_minor_hash) {
                de = (struct ext4_dir_entry_2 *) frames[0].bh->b_data;
                tmp_str.name = de->name;
                tmp_str.len = de->name_len;
                err = ext4_htree_store_dirent(dir_file, 0, 0,
                                              de, &tmp_str);
                if (err != 0)
                        goto errout;
                count++;
        }
        if (start_hash < 2 || (start_hash ==2 && start_minor_hash==0)) {
                de = (struct ext4_dir_entry_2 *) frames[0].bh->b_data;
                de = ext4_next_entry(de, dir->i_sb->s_blocksize);
                tmp_str.name = de->name;
                tmp_str.len = de->name_len;
                err = ext4_htree_store_dirent(dir_file, 2, 0,
                                              de, &tmp_str);
                if (err != 0)
                        goto errout;
                count++;
        }

        while (1) {
                if (fatal_signal_pending(current)) {
                        err = -ERESTARTSYS;
                        goto errout;
                }
                cond_resched();
                block = dx_get_block(frame->at);
                ret = htree_dirblock_to_tree(dir_file, dir, block, &hinfo,
                                             start_hash, start_minor_hash);
                if (ret < 0) {
                        err = ret;
                        goto errout;
                }
                count += ret;
                hashval = ~0;
                ret = ext4_htree_next_block(dir, HASH_NB_ALWAYS,
                                            frame, frames, &hashval);
                *next_hash = hashval;
                if (ret < 0) {
                        err = ret;
                        goto errout;
                }
                /*
                 * Stop if:  (a) there are no more entries, or
                 * (b) we have inserted at least one entry and the
                 * next hash value is not a continuation
                 */
                if ((ret == 0) ||
                    (count && ((hashval & 1) == 0)))
                        break;
        }
        dx_release(frames);
        dxtrace(printk(KERN_DEBUG "Fill tree: returned %d entries, "
                       "next hash: %x\n", count, *next_hash));
        return count;
errout:
        dx_release(frames);
        return (err);
}

static inline int search_dirblock(struct buffer_head *bh,
                                  struct inode *dir,
                                  struct ext4_filename *fname,
                                  unsigned int offset,
                                  struct ext4_dir_entry_2 **res_dir)
{
        return ext4_search_dir(bh, bh->b_data, dir->i_sb->s_blocksize, dir,
                               fname, offset, res_dir);
}

/*
 * Directory block splitting, compacting
 */

/*
 * Create map of hash values, offsets, and sizes, stored at end of block.
 * Returns number of entries mapped.
 */
static int dx_make_map(struct inode *dir, struct buffer_head *bh,
                       struct dx_hash_info *hinfo,
                       struct dx_map_entry *map_tail)
{
        int count = 0;
        struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *)bh->b_data;
        unsigned int buflen = bh->b_size;
        char *base = bh->b_data;
        struct dx_hash_info h = *hinfo;
        int blocksize = EXT4_BLOCK_SIZE(dir->i_sb);

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                buflen -= sizeof(struct ext4_dir_entry_tail);

        while ((char *) de < base + buflen) {
                if (ext4_check_dir_entry(dir, NULL, de, bh, base, buflen,
                                         ((char *)de) - base))
                        return -EFSCORRUPTED;
                if (de->name_len && de->inode) {
                        if (ext4_hash_in_dirent(dir))
                                h.hash = EXT4_DIRENT_HASH(de);
                        else {
                                int err = ext4fs_dirhash(dir, de->name,
                                                     de->name_len, &h);
                                if (err < 0)
                                        return err;
                        }
                        map_tail--;
                        map_tail->hash = h.hash;
                        map_tail->offs = ((char *) de - base)>>2;
                        map_tail->size = ext4_rec_len_from_disk(de->rec_len,
                                                                blocksize);
                        count++;
                        cond_resched();
                }
                de = ext4_next_entry(de, blocksize);
        }
        return count;
}

/* Sort map by hash value */
static void dx_sort_map (struct dx_map_entry *map, unsigned count)
{
        struct dx_map_entry *p, *q, *top = map + count - 1;
        int more;
        /* Combsort until bubble sort doesn't suck */
        while (count > 2) {
                count = count*10/13;
                if (count - 9 < 2) /* 9, 10 -> 11 */
                        count = 11;
                for (p = top, q = p - count; q >= map; p--, q--)
                        if (p->hash < q->hash)
                                swap(*p, *q);
        }
        /* Garden variety bubble sort */
        do {
                more = 0;
                q = top;
                while (q-- > map) {
                        if (q[1].hash >= q[0].hash)
                                continue;
                        swap(*(q+1), *q);
                        more = 1;
                }
        } while(more);
}

static void dx_insert_block(struct dx_frame *frame, u32 hash, ext4_lblk_t block)
{
        struct dx_entry *entries = frame->entries;
        struct dx_entry *old = frame->at, *new = old + 1;
        int count = dx_get_count(entries);

        ASSERT(count < dx_get_limit(entries));
        ASSERT(old < entries + count);
        memmove(new + 1, new, (char *)(entries + count) - (char *)(new));
        dx_set_hash(new, hash);
        dx_set_block(new, block);
        dx_set_count(entries, count + 1);
}

#if IS_ENABLED(CONFIG_UNICODE)
int ext4_fname_setup_ci_filename(struct inode *dir, const struct qstr *iname,
                                  struct ext4_filename *name)
{
        struct qstr *cf_name = &name->cf_name;
        unsigned char *buf;
        struct dx_hash_info *hinfo = &name->hinfo;
        int len;

        if (!IS_CASEFOLDED(dir) ||
            (IS_ENCRYPTED(dir) && !fscrypt_has_encryption_key(dir))) {
                cf_name->name = NULL;
                return 0;
        }

        buf = kmalloc(EXT4_NAME_LEN, GFP_NOFS);
        if (!buf)
                return -ENOMEM;

        len = utf8_casefold(dir->i_sb->s_encoding, iname, buf, EXT4_NAME_LEN);
        if (len <= 0) {
                kfree(buf);
                buf = NULL;
        }
        cf_name->name = buf;
        cf_name->len = (unsigned) len;

        if (!IS_ENCRYPTED(dir))
                return 0;

        hinfo->hash_version = DX_HASH_SIPHASH;
        hinfo->seed = NULL;
        if (cf_name->name)
                return ext4fs_dirhash(dir, cf_name->name, cf_name->len, hinfo);
        else
                return ext4fs_dirhash(dir, iname->name, iname->len, hinfo);
}
#endif

/*
 * Test whether a directory entry matches the filename being searched for.
 *
 * Return: %true if the directory entry matches, otherwise %false.
 */
static bool ext4_match(struct inode *parent,
                              const struct ext4_filename *fname,
                              struct ext4_dir_entry_2 *de)
{
        struct fscrypt_name f;

        if (!de->inode)
                return false;

        f.usr_fname = fname->usr_fname;
        f.disk_name = fname->disk_name;
#ifdef CONFIG_FS_ENCRYPTION
        f.crypto_buf = fname->crypto_buf;
#endif

#if IS_ENABLED(CONFIG_UNICODE)
        if (IS_CASEFOLDED(parent) &&
            (!IS_ENCRYPTED(parent) || fscrypt_has_encryption_key(parent))) {
                /*
                 * Just checking IS_ENCRYPTED(parent) below is not
                 * sufficient to decide whether one can use the hash for
                 * skipping the string comparison, because the key might
                 * have been added right after
                 * ext4_fname_setup_ci_filename().  In this case, a hash
                 * mismatch will be a false negative.  Therefore, make
                 * sure cf_name was properly initialized before
                 * considering the calculated hash.
                 */
                if (sb_no_casefold_compat_fallback(parent->i_sb) &&
                    IS_ENCRYPTED(parent) && fname->cf_name.name &&
                    (fname->hinfo.hash != EXT4_DIRENT_HASH(de) ||
                     fname->hinfo.minor_hash != EXT4_DIRENT_MINOR_HASH(de)))
                        return false;
                /*
                 * Treat comparison errors as not a match.  The
                 * only case where it happens is on a disk
                 * corruption or ENOMEM.
                 */

                return generic_ci_match(parent, fname->usr_fname,
                                        &fname->cf_name, de->name,
                                        de->name_len) > 0;
        }
#endif

        return fscrypt_match_name(&f, de->name, de->name_len);
}

/*
 * Returns 0 if not found, -EFSCORRUPTED on failure, and 1 on success
 */
int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size,
                    struct inode *dir, struct ext4_filename *fname,
                    unsigned int offset, struct ext4_dir_entry_2 **res_dir)
{
        struct ext4_dir_entry_2 * de;
        char * dlimit;
        int de_len;

        de = (struct ext4_dir_entry_2 *)search_buf;
        dlimit = search_buf + buf_size;
        while ((char *) de < dlimit - EXT4_BASE_DIR_LEN) {
                /* this code is executed quadratically often */
                /* do minimal checking `by hand' */
                if (de->name + de->name_len <= dlimit &&
                    ext4_match(dir, fname, de)) {
                        /* found a match - just to be sure, do
                         * a full check */
                        if (ext4_check_dir_entry(dir, NULL, de, bh, search_buf,
                                                 buf_size, offset))
                                return -EFSCORRUPTED;
                        *res_dir = de;
                        return 1;
                }
                /* prevent looping on a bad block */
                de_len = ext4_rec_len_from_disk(de->rec_len,
                                                dir->i_sb->s_blocksize);
                if (de_len <= 0)
                        return -EFSCORRUPTED;
                offset += de_len;
                de = (struct ext4_dir_entry_2 *) ((char *) de + de_len);
        }
        return 0;
}

static int is_dx_internal_node(struct inode *dir, ext4_lblk_t block,
                               struct ext4_dir_entry *de)
{
        struct super_block *sb = dir->i_sb;

        if (!is_dx(dir))
                return 0;
        if (block == 0)
                return 1;
        if (de->inode == 0 &&
            ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize) ==
                        sb->s_blocksize)
                return 1;
        return 0;
}

/*
 *        __ext4_find_entry()
 *
 * finds an entry in the specified directory with the wanted name. It
 * returns the cache buffer in which the entry was found, and the entry
 * itself (as a parameter - res_dir). It does NOT read the inode of the
 * entry - you'll have to do that yourself if you want to.
 *
 * The returned buffer_head has ->b_count elevated.  The caller is expected
 * to brelse() it when appropriate.
 */
static struct buffer_head *__ext4_find_entry(struct inode *dir,
                                             struct ext4_filename *fname,
                                             struct ext4_dir_entry_2 **res_dir,
                                             int *inlined)
{
        struct super_block *sb;
        struct buffer_head *bh_use[NAMEI_RA_SIZE];
        struct buffer_head *bh, *ret = NULL;
        ext4_lblk_t start, block;
        const u8 *name = fname->usr_fname->name;
        size_t ra_max = 0;        /* Number of bh's in the readahead
                                   buffer, bh_use[] */
        size_t ra_ptr = 0;        /* Current index into readahead
                                   buffer */
        ext4_lblk_t  nblocks;
        int i, namelen, retval;

        *res_dir = NULL;
        sb = dir->i_sb;
        namelen = fname->usr_fname->len;
        if (namelen > EXT4_NAME_LEN)
                return NULL;

        if (ext4_has_inline_data(dir)) {
                int has_inline_data = 1;
                ret = ext4_find_inline_entry(dir, fname, res_dir,
                                             &has_inline_data);
                if (inlined)
                        *inlined = has_inline_data;
                if (has_inline_data || IS_ERR(ret))
                        goto cleanup_and_exit;
        }

        if ((namelen <= 2) && (name[0] == '.') &&
            (name[1] == '.' || name[1] == '\0')) {
                /*
                 * "." or ".." will only be in the first block
                 * NFS may look up ".."; "." should be handled by the VFS
                 */
                block = start = 0;
                nblocks = 1;
                goto restart;
        }
        if (is_dx(dir)) {
                ret = ext4_dx_find_entry(dir, fname, res_dir);
                /*
                 * On success, or if the error was file not found,
                 * return.  Otherwise, fall back to doing a search the
                 * old fashioned way.
                 */
                if (IS_ERR(ret) && PTR_ERR(ret) == ERR_BAD_DX_DIR)
                        dxtrace(printk(KERN_DEBUG "ext4_find_entry: dx failed, "
                                       "falling back\n"));
                else if (!sb_no_casefold_compat_fallback(dir->i_sb) &&
                         *res_dir == NULL && IS_CASEFOLDED(dir))
                        dxtrace(printk(KERN_DEBUG "ext4_find_entry: casefold "
                                       "failed, falling back\n"));
                else
                        goto cleanup_and_exit;
                ret = NULL;
        }
        nblocks = dir->i_size >> EXT4_BLOCK_SIZE_BITS(sb);
        if (!nblocks) {
                ret = NULL;
                goto cleanup_and_exit;
        }
        start = EXT4_I(dir)->i_dir_start_lookup;
        if (start >= nblocks)
                start = 0;
        block = start;
restart:
        do {
                /*
                 * We deal with the read-ahead logic here.
                 */
                cond_resched();
                if (ra_ptr >= ra_max) {
                        /* Refill the readahead buffer */
                        ra_ptr = 0;
                        if (block < start)
                                ra_max = start - block;
                        else
                                ra_max = nblocks - block;
                        ra_max = min(ra_max, ARRAY_SIZE(bh_use));
                        retval = ext4_bread_batch(dir, block, ra_max,
                                                  false /* wait */, bh_use);
                        if (retval) {
                                ret = ERR_PTR(retval);
                                ra_max = 0;
                                goto cleanup_and_exit;
                        }
                }
                if ((bh = bh_use[ra_ptr++]) == NULL)
                        goto next;
                wait_on_buffer(bh);
                if (!buffer_uptodate(bh)) {
                        EXT4_ERROR_INODE_ERR(dir, EIO,
                                             "reading directory lblock %lu",
                                             (unsigned long) block);
                        brelse(bh);
                        ret = ERR_PTR(-EIO);
                        goto cleanup_and_exit;
                }
                if (!buffer_verified(bh) &&
                    !is_dx_internal_node(dir, block,
                                         (struct ext4_dir_entry *)bh->b_data) &&
                    !ext4_dirblock_csum_verify(dir, bh)) {
                        EXT4_ERROR_INODE_ERR(dir, EFSBADCRC,
                                             "checksumming directory "
                                             "block %lu", (unsigned long)block);
                        brelse(bh);
                        ret = ERR_PTR(-EFSBADCRC);
                        goto cleanup_and_exit;
                }
                set_buffer_verified(bh);
                i = search_dirblock(bh, dir, fname,
                                    EXT4_LBLK_TO_B(dir, block), res_dir);
                if (i == 1) {
                        EXT4_I(dir)->i_dir_start_lookup = block;
                        ret = bh;
                        goto cleanup_and_exit;
                } else {
                        brelse(bh);
                        if (i < 0) {
                                ret = ERR_PTR(i);
                                goto cleanup_and_exit;
                        }
                }
        next:
                if (++block >= nblocks)
                        block = 0;
        } while (block != start);

        /*
         * If the directory has grown while we were searching, then
         * search the last part of the directory before giving up.
         */
        block = nblocks;
        nblocks = dir->i_size >> EXT4_BLOCK_SIZE_BITS(sb);
        if (block < nblocks) {
                start = 0;
                goto restart;
        }

cleanup_and_exit:
        /* Clean up the read-ahead blocks */
        for (; ra_ptr < ra_max; ra_ptr++)
                brelse(bh_use[ra_ptr]);
        return ret;
}

static struct buffer_head *ext4_find_entry(struct inode *dir,
                                           const struct qstr *d_name,
                                           struct ext4_dir_entry_2 **res_dir,
                                           int *inlined)
{
        int err;
        struct ext4_filename fname;
        struct buffer_head *bh;

        err = ext4_fname_setup_filename(dir, d_name, 1, &fname);
        if (err == -ENOENT)
                return NULL;
        if (err)
                return ERR_PTR(err);

        bh = __ext4_find_entry(dir, &fname, res_dir, inlined);

        ext4_fname_free_filename(&fname);
        return bh;
}

static struct buffer_head *ext4_lookup_entry(struct inode *dir,
                                             struct dentry *dentry,
                                             struct ext4_dir_entry_2 **res_dir)
{
        int err;
        struct ext4_filename fname;
        struct buffer_head *bh;

        err = ext4_fname_prepare_lookup(dir, dentry, &fname);
        if (err == -ENOENT)
                return NULL;
        if (err)
                return ERR_PTR(err);

        bh = __ext4_find_entry(dir, &fname, res_dir, NULL);

        ext4_fname_free_filename(&fname);
        return bh;
}

static struct buffer_head * ext4_dx_find_entry(struct inode *dir,
                        struct ext4_filename *fname,
                        struct ext4_dir_entry_2 **res_dir)
{
        struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
        struct buffer_head *bh;
        ext4_lblk_t block;
        int retval;

#ifdef CONFIG_FS_ENCRYPTION
        *res_dir = NULL;
#endif
        frame = dx_probe(fname, dir, NULL, frames);
        if (IS_ERR(frame))
                return ERR_CAST(frame);
        do {
                block = dx_get_block(frame->at);
                bh = ext4_read_dirblock(dir, block, DIRENT_HTREE);
                if (IS_ERR(bh))
                        goto errout;

                retval = search_dirblock(bh, dir, fname,
                                         EXT4_LBLK_TO_B(dir, block), res_dir);
                if (retval == 1)
                        goto success;
                brelse(bh);
                if (retval < 0) {
                        bh = ERR_PTR(ERR_BAD_DX_DIR);
                        goto errout;
                }

                /* Check to see if we should continue to search */
                retval = ext4_htree_next_block(dir, fname->hinfo.hash, frame,
                                               frames, NULL);
                if (retval < 0) {
                        ext4_warning_inode(dir,
                                "error %d reading directory index block",
                                retval);
                        bh = ERR_PTR(retval);
                        goto errout;
                }
        } while (retval == 1);

        bh = NULL;
errout:
        dxtrace(printk(KERN_DEBUG "%s not found\n", fname->usr_fname->name));
success:
        dx_release(frames);
        return bh;
}

static struct dentry *ext4_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
{
        struct inode *inode;
        struct ext4_dir_entry_2 *de = NULL;
        struct buffer_head *bh;

        if (dentry->d_name.len > EXT4_NAME_LEN)
                return ERR_PTR(-ENAMETOOLONG);

        bh = ext4_lookup_entry(dir, dentry, &de);
        if (IS_ERR(bh))
                return ERR_CAST(bh);
        inode = NULL;
        if (bh) {
                __u32 ino = le32_to_cpu(de->inode);
                brelse(bh);
                if (!ext4_valid_inum(dir->i_sb, ino)) {
                        EXT4_ERROR_INODE(dir, "bad inode number: %u", ino);
                        return ERR_PTR(-EFSCORRUPTED);
                }
                if (unlikely(ino == dir->i_ino)) {
                        EXT4_ERROR_INODE(dir, "'%pd' linked to parent dir",
                                         dentry);
                        return ERR_PTR(-EFSCORRUPTED);
                }
                inode = ext4_iget(dir->i_sb, ino, EXT4_IGET_NORMAL);
                if (inode == ERR_PTR(-ESTALE)) {
                        EXT4_ERROR_INODE(dir,
                                         "deleted inode referenced: %u",
                                         ino);
                        return ERR_PTR(-EFSCORRUPTED);
                }
                if (!IS_ERR(inode) && IS_ENCRYPTED(dir) &&
                    (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) &&
                    !fscrypt_has_permitted_context(dir, inode)) {
                        ext4_warning(inode->i_sb,
                                     "Inconsistent encryption contexts: %llu/%llu",
                                     dir->i_ino, inode->i_ino);
                        iput(inode);
                        return ERR_PTR(-EPERM);
                }
        }

        if (IS_ENABLED(CONFIG_UNICODE) && !inode && IS_CASEFOLDED(dir)) {
                /* Eventually we want to call d_add_ci(dentry, NULL)
                 * for negative dentries in the encoding case as
                 * well.  For now, prevent the negative dentry
                 * from being cached.
                 */
                return NULL;
        }

        return d_splice_alias(inode, dentry);
}


struct dentry *ext4_get_parent(struct dentry *child)
{
        __u32 ino;
        struct ext4_dir_entry_2 * de = NULL;
        struct buffer_head *bh;

        bh = ext4_find_entry(d_inode(child), &dotdot_name, &de, NULL);
        if (IS_ERR(bh))
                return ERR_CAST(bh);
        if (!bh)
                return ERR_PTR(-ENOENT);
        ino = le32_to_cpu(de->inode);
        brelse(bh);

        if (!ext4_valid_inum(child->d_sb, ino)) {
                EXT4_ERROR_INODE(d_inode(child),
                                 "bad parent inode number: %u", ino);
                return ERR_PTR(-EFSCORRUPTED);
        }

        return d_obtain_alias(ext4_iget(child->d_sb, ino, EXT4_IGET_NORMAL));
}

/*
 * Move count entries from end of map between two memory locations.
 * Returns pointer to last entry moved.
 */
static struct ext4_dir_entry_2 *
dx_move_dirents(struct inode *dir, char *from, char *to,
                struct dx_map_entry *map, int count,
                unsigned blocksize)
{
        unsigned rec_len = 0;

        while (count--) {
                struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *)
                                                (from + (map->offs<<2));
                rec_len = ext4_dir_rec_len(de->name_len, dir);

                memcpy (to, de, rec_len);
                ((struct ext4_dir_entry_2 *) to)->rec_len =
                                ext4_rec_len_to_disk(rec_len, blocksize);

                /* wipe dir_entry excluding the rec_len field */
                de->inode = 0;
                memset(&de->name_len, 0, ext4_rec_len_from_disk(de->rec_len,
                                                                blocksize) -
                                         offsetof(struct ext4_dir_entry_2,
                                                                name_len));

                map++;
                to += rec_len;
        }
        return (struct ext4_dir_entry_2 *) (to - rec_len);
}

/*
 * Compact each dir entry in the range to the minimal rec_len.
 * Returns pointer to last entry in range.
 */
static struct ext4_dir_entry_2 *dx_pack_dirents(struct inode *dir, char *base,
                                                        unsigned int blocksize)
{
        struct ext4_dir_entry_2 *next, *to, *prev, *de = (struct ext4_dir_entry_2 *) base;
        unsigned rec_len = 0;

        prev = to = de;
        while ((char*)de < base + blocksize) {
                next = ext4_next_entry(de, blocksize);
                if (de->inode && de->name_len) {
                        rec_len = ext4_dir_rec_len(de->name_len, dir);
                        if (de > to)
                                memmove(to, de, rec_len);
                        to->rec_len = ext4_rec_len_to_disk(rec_len, blocksize);
                        prev = to;
                        to = (struct ext4_dir_entry_2 *) (((char *) to) + rec_len);
                }
                de = next;
        }
        return prev;
}

/*
 * Split a full leaf block to make room for a new dir entry.
 * Allocate a new block, and move entries so that they are approx. equally full.
 * Returns pointer to de in block into which the new entry will be inserted.
 */
static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir,
                        struct buffer_head **bh,struct dx_frame *frame,
                        struct dx_hash_info *hinfo)
{
        unsigned blocksize = dir->i_sb->s_blocksize;
        unsigned continued;
        int count;
        struct buffer_head *bh2;
        ext4_lblk_t newblock;
        u32 hash2;
        struct dx_map_entry *map;
        char *data1 = (*bh)->b_data, *data2;
        unsigned split, move, size;
        struct ext4_dir_entry_2 *de = NULL, *de2;
        int        csum_size = 0;
        int        err = 0, i;

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        bh2 = ext4_append(handle, dir, &newblock);
        if (IS_ERR(bh2)) {
                brelse(*bh);
                *bh = NULL;
                return ERR_CAST(bh2);
        }

        BUFFER_TRACE(*bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, dir->i_sb, *bh,
                                            EXT4_JTR_NONE);
        if (err)
                goto journal_error;

        BUFFER_TRACE(frame->bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, dir->i_sb, frame->bh,
                                            EXT4_JTR_NONE);
        if (err)
                goto journal_error;

        data2 = bh2->b_data;

        /* create map in the end of data2 block */
        map = (struct dx_map_entry *) (data2 + blocksize);
        count = dx_make_map(dir, *bh, hinfo, map);
        if (count < 0) {
                err = count;
                goto journal_error;
        }
        map -= count;
        dx_sort_map(map, count);
        /* Ensure that neither split block is over half full */
        size = 0;
        move = 0;
        for (i = count-1; i >= 0; i--) {
                /* is more than half of this entry in 2nd half of the block? */
                if (size + map[i].size/2 > blocksize/2)
                        break;
                size += map[i].size;
                move++;
        }
        /*
         * map index at which we will split
         *
         * If the sum of active entries didn't exceed half the block size, just
         * split it in half by count; each resulting block will have at least
         * half the space free.
         */
        if (i >= 0)
                split = count - move;
        else
                split = count/2;

        if (WARN_ON_ONCE(split == 0)) {
                /* Should never happen, but avoid out-of-bounds access below */
                ext4_error_inode_block(dir, (*bh)->b_blocknr, 0,
                        "bad indexed directory? hash=%08x:%08x count=%d move=%u",
                        hinfo->hash, hinfo->minor_hash, count, move);
                err = -EFSCORRUPTED;
                goto out;
        }

        hash2 = map[split].hash;
        continued = hash2 == map[split - 1].hash;
        dxtrace(printk(KERN_INFO "Split block %lu at %x, %i/%i\n",
                        (unsigned long)dx_get_block(frame->at),
                                        hash2, split, count-split));

        /* Fancy dance to stay within two buffers */
        de2 = dx_move_dirents(dir, data1, data2, map + split, count - split,
                              blocksize);
        de = dx_pack_dirents(dir, data1, blocksize);
        de->rec_len = ext4_rec_len_to_disk(data1 + (blocksize - csum_size) -
                                           (char *) de,
                                           blocksize);
        de2->rec_len = ext4_rec_len_to_disk(data2 + (blocksize - csum_size) -
                                            (char *) de2,
                                            blocksize);
        if (csum_size) {
                ext4_initialize_dirent_tail(*bh, blocksize);
                ext4_initialize_dirent_tail(bh2, blocksize);
        }

        dxtrace(dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *) data1,
                        blocksize, 1));
        dxtrace(dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *) data2,
                        blocksize, 1));

        /* Which block gets the new entry? */
        if (hinfo->hash >= hash2) {
                swap(*bh, bh2);
                de = de2;
        }
        dx_insert_block(frame, hash2 + continued, newblock);
        err = ext4_handle_dirty_dirblock(handle, dir, bh2);
        if (err)
                goto journal_error;
        err = ext4_handle_dirty_dx_node(handle, dir, frame->bh);
        if (err)
                goto journal_error;
        brelse(bh2);
        dxtrace(dx_show_index("frame", frame->entries));
        return de;

journal_error:
        ext4_std_error(dir->i_sb, err);
out:
        brelse(*bh);
        brelse(bh2);
        *bh = NULL;
        return ERR_PTR(err);
}

int ext4_find_dest_de(struct inode *dir, struct buffer_head *bh,
                      void *buf, int buf_size,
                      struct ext4_filename *fname,
                      struct ext4_dir_entry_2 **dest_de)
{
        struct ext4_dir_entry_2 *de;
        unsigned short reclen = ext4_dir_rec_len(fname_len(fname), dir);
        int nlen, rlen;
        unsigned int offset = 0;
        char *top;

        de = buf;
        top = buf + buf_size - reclen;
        while ((char *) de <= top) {
                if (ext4_check_dir_entry(dir, NULL, de, bh,
                                         buf, buf_size, offset))
                        return -EFSCORRUPTED;
                if (ext4_match(dir, fname, de))
                        return -EEXIST;
                nlen = ext4_dir_rec_len(de->name_len, dir);
                rlen = ext4_rec_len_from_disk(de->rec_len, buf_size);
                if ((de->inode ? rlen - nlen : rlen) >= reclen)
                        break;
                de = (struct ext4_dir_entry_2 *)((char *)de + rlen);
                offset += rlen;
        }
        if ((char *) de > top)
                return -ENOSPC;

        *dest_de = de;
        return 0;
}

void ext4_insert_dentry(struct inode *dir,
                        struct inode *inode,
                        struct ext4_dir_entry_2 *de,
                        int buf_size,
                        struct ext4_filename *fname)
{

        int nlen, rlen;

        nlen = ext4_dir_rec_len(de->name_len, dir);
        rlen = ext4_rec_len_from_disk(de->rec_len, buf_size);
        if (de->inode) {
                struct ext4_dir_entry_2 *de1 =
                        (struct ext4_dir_entry_2 *)((char *)de + nlen);
                de1->rec_len = ext4_rec_len_to_disk(rlen - nlen, buf_size);
                de->rec_len = ext4_rec_len_to_disk(nlen, buf_size);
                de = de1;
        }
        de->file_type = EXT4_FT_UNKNOWN;
        de->inode = cpu_to_le32(inode->i_ino);
        ext4_set_de_type(inode->i_sb, de, inode->i_mode);
        de->name_len = fname_len(fname);
        memcpy(de->name, fname_name(fname), fname_len(fname));
        if (ext4_hash_in_dirent(dir)) {
                struct dx_hash_info *hinfo = &fname->hinfo;

                EXT4_DIRENT_HASHES(de)->hash = cpu_to_le32(hinfo->hash);
                EXT4_DIRENT_HASHES(de)->minor_hash =
                                                cpu_to_le32(hinfo->minor_hash);
        }
}

/*
 * Add a new entry into a directory (leaf) block.  If de is non-NULL,
 * it points to a directory entry which is guaranteed to be large
 * enough for new directory entry.  If de is NULL, then
 * add_dirent_to_buf will attempt search the directory block for
 * space.  It will return -ENOSPC if no space is available, and -EIO
 * and -EEXIST if directory entry already exists.
 */
static int add_dirent_to_buf(handle_t *handle, struct ext4_filename *fname,
                             struct inode *dir,
                             struct inode *inode, struct ext4_dir_entry_2 *de,
                             struct buffer_head *bh)
{
        unsigned int        blocksize = dir->i_sb->s_blocksize;
        int                csum_size = 0;
        int                err, err2;

        if (ext4_has_feature_metadata_csum(inode->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        if (!de) {
                err = ext4_find_dest_de(dir, bh, bh->b_data,
                                        blocksize - csum_size, fname, &de);
                if (err)
                        return err;
        }
        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, dir->i_sb, bh,
                                            EXT4_JTR_NONE);
        if (err) {
                ext4_std_error(dir->i_sb, err);
                return err;
        }

        /* By now the buffer is marked for journaling */
        ext4_insert_dentry(dir, inode, de, blocksize, fname);

        /*
         * XXX shouldn't update any times until successful
         * completion of syscall, but too many callers depend
         * on this.
         *
         * XXX similarly, too many callers depend on
         * ext4_new_inode() setting the times, but error
         * recovery deletes the inode, so the worst that can
         * happen is that the times are slightly out of date
         * and/or different from the directory change time.
         */
        inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
        ext4_update_dx_flag(dir);
        inode_inc_iversion(dir);
        err2 = ext4_mark_inode_dirty(handle, dir);
        BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
        err = ext4_handle_dirty_dirblock(handle, dir, bh);
        if (err)
                ext4_std_error(dir->i_sb, err);
        return err ? err : err2;
}

static bool ext4_check_dx_root(struct inode *dir, struct dx_root *root)
{
        struct fake_dirent *fde;
        const char *error_msg;
        unsigned int rlen;
        unsigned int blocksize = dir->i_sb->s_blocksize;
        char *blockend = (char *)root + dir->i_sb->s_blocksize;

        fde = &root->dot;
        if (unlikely(fde->name_len != 1)) {
                error_msg = "invalid name_len for '.'";
                goto corrupted;
        }
        if (unlikely(strncmp(root->dot_name, ".", fde->name_len))) {
                error_msg = "invalid name for '.'";
                goto corrupted;
        }
        rlen = ext4_rec_len_from_disk(fde->rec_len, blocksize);
        if (unlikely((char *)fde + rlen >= blockend)) {
                error_msg = "invalid rec_len for '.'";
                goto corrupted;
        }

        fde = &root->dotdot;
        if (unlikely(fde->name_len != 2)) {
                error_msg = "invalid name_len for '..'";
                goto corrupted;
        }
        if (unlikely(strncmp(root->dotdot_name, "..", fde->name_len))) {
                error_msg = "invalid name for '..'";
                goto corrupted;
        }
        rlen = ext4_rec_len_from_disk(fde->rec_len, blocksize);
        if (unlikely((char *)fde + rlen >= blockend)) {
                error_msg = "invalid rec_len for '..'";
                goto corrupted;
        }

        return true;

corrupted:
        EXT4_ERROR_INODE(dir, "Corrupt dir, %s, running e2fsck is recommended",
                         error_msg);
        return false;
}

/*
 * This converts a one block unindexed directory to a 3 block indexed
 * directory, and adds the dentry to the indexed directory.
 */
static int make_indexed_dir(handle_t *handle, struct ext4_filename *fname,
                            struct inode *dir,
                            struct inode *inode, struct buffer_head *bh)
{
        struct buffer_head *bh2;
        struct dx_root        *root;
        struct dx_frame        frames[EXT4_HTREE_LEVEL], *frame;
        struct dx_entry *entries;
        struct ext4_dir_entry_2        *de, *de2;
        char                *data2, *top;
        unsigned        len;
        int                retval;
        unsigned        blocksize;
        ext4_lblk_t  block;
        struct fake_dirent *fde;
        int csum_size = 0;

        if (ext4_has_feature_metadata_csum(inode->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        blocksize =  dir->i_sb->s_blocksize;
        dxtrace(printk(KERN_DEBUG "Creating index: inode %llu\n", dir->i_ino));
        BUFFER_TRACE(bh, "get_write_access");
        retval = ext4_journal_get_write_access(handle, dir->i_sb, bh,
                                               EXT4_JTR_NONE);
        if (retval) {
                ext4_std_error(dir->i_sb, retval);
                brelse(bh);
                return retval;
        }

        root = (struct dx_root *) bh->b_data;
        if (!ext4_check_dx_root(dir, root)) {
                brelse(bh);
                return -EFSCORRUPTED;
        }

        /* The 0th block becomes the root, move the dirents out */
        fde = &root->dotdot;
        de = (struct ext4_dir_entry_2 *)((char *)fde +
                ext4_rec_len_from_disk(fde->rec_len, blocksize));
        len = ((char *) root) + (blocksize - csum_size) - (char *) de;

        /* Allocate new block for the 0th block's dirents */
        bh2 = ext4_append(handle, dir, &block);
        if (IS_ERR(bh2)) {
                brelse(bh);
                return PTR_ERR(bh2);
        }
        ext4_set_inode_flag(dir, EXT4_INODE_INDEX);
        data2 = bh2->b_data;

        memcpy(data2, de, len);
        memset(de, 0, len); /* wipe old data */
        de = (struct ext4_dir_entry_2 *) data2;
        top = data2 + len;
        while ((char *)(de2 = ext4_next_entry(de, blocksize)) < top) {
                if (ext4_check_dir_entry(dir, NULL, de, bh2, data2, len,
                                        (char *)de - data2)) {
                        brelse(bh2);
                        brelse(bh);
                        return -EFSCORRUPTED;
                }
                de = de2;
        }
        de->rec_len = ext4_rec_len_to_disk(data2 + (blocksize - csum_size) -
                                           (char *) de, blocksize);

        if (csum_size)
                ext4_initialize_dirent_tail(bh2, blocksize);

        /* Initialize the root; the dot dirents already exist */
        de = (struct ext4_dir_entry_2 *) (&root->dotdot);
        de->rec_len = ext4_rec_len_to_disk(
                        blocksize - ext4_dir_rec_len(2, NULL), blocksize);
        memset (&root->info, 0, sizeof(root->info));
        root->info.info_length = sizeof(root->info);
        if (ext4_hash_in_dirent(dir))
                root->info.hash_version = DX_HASH_SIPHASH;
        else
                root->info.hash_version =
                                EXT4_SB(dir->i_sb)->s_def_hash_version;

        entries = root->entries;
        dx_set_block(entries, 1);
        dx_set_count(entries, 1);
        dx_set_limit(entries, dx_root_limit(dir, sizeof(root->info)));

        /* Initialize as for dx_probe */
        fname->hinfo.hash_version = root->info.hash_version;
        if (fname->hinfo.hash_version <= DX_HASH_TEA)
                fname->hinfo.hash_version += EXT4_SB(dir->i_sb)->s_hash_unsigned;
        fname->hinfo.seed = EXT4_SB(dir->i_sb)->s_hash_seed;

        /* casefolded encrypted hashes are computed on fname setup */
        if (!ext4_hash_in_dirent(dir)) {
                int err = ext4fs_dirhash(dir, fname_name(fname),
                                         fname_len(fname), &fname->hinfo);
                if (err < 0) {
                        brelse(bh2);
                        brelse(bh);
                        return err;
                }
        }
        memset(frames, 0, sizeof(frames));
        frame = frames;
        frame->entries = entries;
        frame->at = entries;
        frame->bh = bh;

        retval = ext4_handle_dirty_dx_node(handle, dir, frame->bh);
        if (retval)
                goto out_frames;
        retval = ext4_handle_dirty_dirblock(handle, dir, bh2);
        if (retval)
                goto out_frames;

        de = do_split(handle,dir, &bh2, frame, &fname->hinfo);
        if (IS_ERR(de)) {
                retval = PTR_ERR(de);
                goto out_frames;
        }

        retval = add_dirent_to_buf(handle, fname, dir, inode, de, bh2);
out_frames:
        /*
         * Even if the block split failed, we have to properly write
         * out all the changes we did so far. Otherwise we can end up
         * with corrupted filesystem.
         */
        if (retval)
                ext4_mark_inode_dirty(handle, dir);
        dx_release(frames);
        brelse(bh2);
        return retval;
}

/*
 *        ext4_add_entry()
 *
 * adds a file entry to the specified directory, using the same
 * semantics as ext4_find_entry(). It returns NULL if it failed.
 *
 * NOTE!! The inode part of 'de' is left at 0 - which means you
 * may not sleep between calling this and putting something into
 * the entry, as someone else might have used it while you slept.
 */
static int __ext4_add_entry(handle_t *handle, struct inode *dir,
                          const struct qstr *d_name,
                          struct inode *inode)
{
        struct buffer_head *bh = NULL;
        struct ext4_dir_entry_2 *de;
        struct super_block *sb;
        struct ext4_filename fname;
        int        retval;
        int        dx_fallback=0;
        unsigned blocksize;
        ext4_lblk_t block, blocks;
        int        csum_size = 0;

        if (ext4_has_feature_metadata_csum(inode->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        sb = dir->i_sb;
        blocksize = sb->s_blocksize;

        if (!generic_ci_validate_strict_name(dir, d_name))
                return -EINVAL;

        retval = ext4_fname_setup_filename(dir, d_name, 0, &fname);
        if (retval)
                return retval;

        if (ext4_has_inline_data(dir)) {
                retval = ext4_try_add_inline_entry(handle, &fname, dir, inode);
                if (retval < 0)
                        goto out;
                if (retval == 1) {
                        retval = 0;
                        goto out;
                }
        }

        if (is_dx(dir)) {
                retval = ext4_dx_add_entry(handle, &fname, dir, inode);
                if (!retval || (retval != ERR_BAD_DX_DIR))
                        goto out;
                /* Can we just ignore htree data? */
                if (ext4_has_feature_metadata_csum(sb)) {
                        EXT4_ERROR_INODE(dir,
                                "Directory has corrupted htree index.");
                        retval = -EFSCORRUPTED;
                        goto out;
                }
                ext4_clear_inode_flag(dir, EXT4_INODE_INDEX);
                dx_fallback++;
                retval = ext4_mark_inode_dirty(handle, dir);
                if (unlikely(retval))
                        goto out;
        }
        blocks = dir->i_size >> sb->s_blocksize_bits;
        for (block = 0; block < blocks; block++) {
                bh = ext4_read_dirblock(dir, block, DIRENT);
                if (bh == NULL) {
                        bh = ext4_bread(handle, dir, block,
                                        EXT4_GET_BLOCKS_CREATE);
                        goto add_to_new_block;
                }
                if (IS_ERR(bh)) {
                        retval = PTR_ERR(bh);
                        bh = NULL;
                        goto out;
                }
                retval = add_dirent_to_buf(handle, &fname, dir, inode,
                                           NULL, bh);
                if (retval != -ENOSPC)
                        goto out;

                if (blocks == 1 && !dx_fallback &&
                    ext4_has_feature_dir_index(sb)) {
                        retval = make_indexed_dir(handle, &fname, dir,
                                                  inode, bh);
                        bh = NULL; /* make_indexed_dir releases bh */
                        goto out;
                }
                brelse(bh);
        }
        bh = ext4_append(handle, dir, &block);
add_to_new_block:
        if (IS_ERR(bh)) {
                retval = PTR_ERR(bh);
                bh = NULL;
                goto out;
        }
        de = (struct ext4_dir_entry_2 *) bh->b_data;
        de->inode = 0;
        de->rec_len = ext4_rec_len_to_disk(blocksize - csum_size, blocksize);

        if (csum_size)
                ext4_initialize_dirent_tail(bh, blocksize);

        retval = add_dirent_to_buf(handle, &fname, dir, inode, de, bh);
out:
        ext4_fname_free_filename(&fname);
        brelse(bh);
        if (retval == 0)
                ext4_set_inode_state(inode, EXT4_STATE_NEWENTRY);
        return retval;
}

static int ext4_add_entry(handle_t *handle, struct dentry *dentry,
                          struct inode *inode)
{
        struct inode *dir = d_inode(dentry->d_parent);

        if (fscrypt_is_nokey_name(dentry))
                return -ENOKEY;
        return __ext4_add_entry(handle, dir, &dentry->d_name, inode);
}

/*
 * Returns 0 for success, or a negative error value
 */
static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname,
                             struct inode *dir, struct inode *inode)
{
        struct dx_frame frames[EXT4_HTREE_LEVEL], *frame;
        struct dx_entry *entries, *at;
        struct buffer_head *bh;
        struct super_block *sb = dir->i_sb;
        struct ext4_dir_entry_2 *de;
        int restart;
        int err;

again:
        restart = 0;
        frame = dx_probe(fname, dir, NULL, frames);
        if (IS_ERR(frame))
                return PTR_ERR(frame);
        entries = frame->entries;
        at = frame->at;
        bh = ext4_read_dirblock(dir, dx_get_block(frame->at), DIRENT_HTREE);
        if (IS_ERR(bh)) {
                err = PTR_ERR(bh);
                bh = NULL;
                goto cleanup;
        }

        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, sb, bh, EXT4_JTR_NONE);
        if (err)
                goto journal_error;

        err = add_dirent_to_buf(handle, fname, dir, inode, NULL, bh);
        if (err != -ENOSPC)
                goto cleanup;

        err = 0;
        /* Block full, should compress but for now just split */
        dxtrace(printk(KERN_DEBUG "using %u of %u node entries\n",
                       dx_get_count(entries), dx_get_limit(entries)));
        /* Need to split index? */
        if (dx_get_count(entries) == dx_get_limit(entries)) {
                ext4_lblk_t newblock;
                int levels = frame - frames + 1;
                unsigned int icount;
                int add_level = 1;
                struct dx_entry *entries2;
                struct dx_node *node2;
                struct buffer_head *bh2;

                while (frame > frames) {
                        if (dx_get_count((frame - 1)->entries) <
                            dx_get_limit((frame - 1)->entries)) {
                                add_level = 0;
                                break;
                        }
                        frame--; /* split higher index block */
                        at = frame->at;
                        entries = frame->entries;
                        restart = 1;
                }
                if (add_level && levels == ext4_dir_htree_level(sb)) {
                        ext4_warning(sb, "Directory (ino: %llu) index full, "
                                         "reach max htree level :%d",
                                         dir->i_ino, levels);
                        if (ext4_dir_htree_level(sb) < EXT4_HTREE_LEVEL) {
                                ext4_warning(sb, "Large directory feature is "
                                                 "not enabled on this "
                                                 "filesystem");
                        }
                        err = -ENOSPC;
                        goto cleanup;
                }
                icount = dx_get_count(entries);
                bh2 = ext4_append(handle, dir, &newblock);
                if (IS_ERR(bh2)) {
                        err = PTR_ERR(bh2);
                        goto cleanup;
                }
                node2 = (struct dx_node *)(bh2->b_data);
                entries2 = node2->entries;
                memset(&node2->fake, 0, sizeof(struct fake_dirent));
                node2->fake.rec_len = ext4_rec_len_to_disk(sb->s_blocksize,
                                                           sb->s_blocksize);
                BUFFER_TRACE(frame->bh, "get_write_access");
                err = ext4_journal_get_write_access(handle, sb, frame->bh,
                                                    EXT4_JTR_NONE);
                if (err) {
                        brelse(bh2);
                        goto journal_error;
                }
                if (!add_level) {
                        unsigned icount1 = icount/2, icount2 = icount - icount1;
                        unsigned hash2 = dx_get_hash(entries + icount1);
                        dxtrace(printk(KERN_DEBUG "Split index %i/%i\n",
                                       icount1, icount2));

                        BUFFER_TRACE(frame->bh, "get_write_access"); /* index root */
                        err = ext4_journal_get_write_access(handle, sb,
                                                            (frame - 1)->bh,
                                                            EXT4_JTR_NONE);
                        if (err) {
                                brelse(bh2);
                                goto journal_error;
                        }

                        memcpy((char *) entries2, (char *) (entries + icount1),
                               icount2 * sizeof(struct dx_entry));
                        dx_set_count(entries, icount1);
                        dx_set_count(entries2, icount2);
                        dx_set_limit(entries2, dx_node_limit(dir));

                        /* Which index block gets the new entry? */
                        if (at - entries >= icount1) {
                                frame->at = at - entries - icount1 + entries2;
                                frame->entries = entries = entries2;
                                swap(frame->bh, bh2);
                        }
                        dx_insert_block((frame - 1), hash2, newblock);
                        dxtrace(dx_show_index("node", frame->entries));
                        dxtrace(dx_show_index("node",
                               ((struct dx_node *) bh2->b_data)->entries));
                        err = ext4_handle_dirty_dx_node(handle, dir, bh2);
                        if (err) {
                                brelse(bh2);
                                goto journal_error;
                        }
                        brelse (bh2);
                        err = ext4_handle_dirty_dx_node(handle, dir,
                                                   (frame - 1)->bh);
                        if (err)
                                goto journal_error;
                        err = ext4_handle_dirty_dx_node(handle, dir,
                                                        frame->bh);
                        if (restart || err)
                                goto journal_error;
                } else {
                        struct dx_root *dxroot;
                        memcpy((char *) entries2, (char *) entries,
                               icount * sizeof(struct dx_entry));
                        dx_set_limit(entries2, dx_node_limit(dir));

                        /* Set up root */
                        dx_set_count(entries, 1);
                        dx_set_block(entries + 0, newblock);
                        dxroot = (struct dx_root *)frames[0].bh->b_data;
                        dxroot->info.indirect_levels += 1;
                        dxtrace(printk(KERN_DEBUG
                                       "Creating %d level index...\n",
                                       dxroot->info.indirect_levels));
                        err = ext4_handle_dirty_dx_node(handle, dir, frame->bh);
                        if (err) {
                                brelse(bh2);
                                goto journal_error;
                        }
                        err = ext4_handle_dirty_dx_node(handle, dir, bh2);
                        brelse(bh2);
                        restart = 1;
                        goto journal_error;
                }
        }
        de = do_split(handle, dir, &bh, frame, &fname->hinfo);
        if (IS_ERR(de)) {
                err = PTR_ERR(de);
                goto cleanup;
        }
        err = add_dirent_to_buf(handle, fname, dir, inode, de, bh);
        goto cleanup;

journal_error:
        ext4_std_error(dir->i_sb, err); /* this is a no-op if err == 0 */
cleanup:
        brelse(bh);
        dx_release(frames);
        /* @restart is true means htree-path has been changed, we need to
         * repeat dx_probe() to find out valid htree-path
         */
        if (restart && err == 0)
                goto again;
        return err;
}

/*
 * ext4_generic_delete_entry deletes a directory entry by merging it
 * with the previous entry
 */
int ext4_generic_delete_entry(struct inode *dir,
                              struct ext4_dir_entry_2 *de_del,
                              struct buffer_head *bh,
                              void *entry_buf,
                              int buf_size,
                              int csum_size)
{
        struct ext4_dir_entry_2 *de, *pde;
        unsigned int blocksize = dir->i_sb->s_blocksize;
        int i;

        i = 0;
        pde = NULL;
        de = entry_buf;
        while (i < buf_size - csum_size) {
                if (ext4_check_dir_entry(dir, NULL, de, bh,
                                         entry_buf, buf_size, i))
                        return -EFSCORRUPTED;
                if (de == de_del)  {
                        if (pde) {
                                pde->rec_len = ext4_rec_len_to_disk(
                                        ext4_rec_len_from_disk(pde->rec_len,
                                                               blocksize) +
                                        ext4_rec_len_from_disk(de->rec_len,
                                                               blocksize),
                                        blocksize);

                                /* wipe entire dir_entry */
                                memset(de, 0, ext4_rec_len_from_disk(de->rec_len,
                                                                blocksize));
                        } else {
                                /* wipe dir_entry excluding the rec_len field */
                                de->inode = 0;
                                memset(&de->name_len, 0,
                                        ext4_rec_len_from_disk(de->rec_len,
                                                                blocksize) -
                                        offsetof(struct ext4_dir_entry_2,
                                                                name_len));
                        }

                        inode_inc_iversion(dir);
                        return 0;
                }
                i += ext4_rec_len_from_disk(de->rec_len, blocksize);
                pde = de;
                de = ext4_next_entry(de, blocksize);
        }
        return -ENOENT;
}

static int ext4_delete_entry(handle_t *handle,
                             struct inode *dir,
                             struct ext4_dir_entry_2 *de_del,
                             struct buffer_head *bh)
{
        int err, csum_size = 0;

        if (ext4_has_inline_data(dir)) {
                int has_inline_data = 1;
                err = ext4_delete_inline_entry(handle, dir, de_del, bh,
                                               &has_inline_data);
                if (has_inline_data)
                        return err;
        }

        if (ext4_has_feature_metadata_csum(dir->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, dir->i_sb, bh,
                                            EXT4_JTR_NONE);
        if (unlikely(err))
                goto out;

        err = ext4_generic_delete_entry(dir, de_del, bh, bh->b_data,
                                        dir->i_sb->s_blocksize, csum_size);
        if (err)
                goto out;

        BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
        err = ext4_handle_dirty_dirblock(handle, dir, bh);
        if (unlikely(err))
                goto out;

        return 0;
out:
        if (err != -ENOENT)
                ext4_std_error(dir->i_sb, err);
        return err;
}

/*
 * Set directory link count to 1 if nlinks > EXT4_LINK_MAX, or if nlinks == 2
 * since this indicates that nlinks count was previously 1 to avoid overflowing
 * the 16-bit i_links_count field on disk.  Directories with i_nlink == 1 mean
 * that subdirectory link counts are not being maintained accurately.
 *
 * The caller has already checked for i_nlink overflow in case the DIR_LINK
 * feature is not enabled and returned -EMLINK.  The is_dx() check is a proxy
 * for checking S_ISDIR(inode) (since the INODE_INDEX feature will not be set
 * on regular files) and to avoid creating huge/slow non-HTREE directories.
 */
static void ext4_inc_count(struct inode *inode)
{
        inc_nlink(inode);
        if (is_dx(inode) &&
            (inode->i_nlink > EXT4_LINK_MAX || inode->i_nlink == 2))
                set_nlink(inode, 1);
}

/*
 * If a directory had nlink == 1, then we should let it be 1. This indicates
 * directory has >EXT4_LINK_MAX subdirs.
 */
static void ext4_dec_count(struct inode *inode)
{
        if (!S_ISDIR(inode->i_mode) || inode->i_nlink > 2)
                drop_nlink(inode);
}


/*
 * Add non-directory inode to a directory. On success, the inode reference is
 * consumed by dentry is instantiation. This is also indicated by clearing of
 * *inodep pointer. On failure, the caller is responsible for dropping the
 * inode reference in the safe context.
 */
static int ext4_add_nondir(handle_t *handle,
                struct dentry *dentry, struct inode **inodep)
{
        struct inode *dir = d_inode(dentry->d_parent);
        struct inode *inode = *inodep;
        int err = ext4_add_entry(handle, dentry, inode);
        if (!err) {
                err = ext4_mark_inode_dirty(handle, inode);
                if (IS_DIRSYNC(dir))
                        ext4_handle_sync(handle);
                d_instantiate_new(dentry, inode);
                *inodep = NULL;
                return err;
        }
        drop_nlink(inode);
        ext4_mark_inode_dirty(handle, inode);
        ext4_orphan_add(handle, inode);
        unlock_new_inode(inode);
        return err;
}

/*
 * By the time this is called, we already have created
 * the directory cache entry for the new file, but it
 * is so far negative - it has no inode.
 *
 * If the create succeeds, we fill in the inode information
 * with d_instantiate().
 */
static int ext4_create(struct mnt_idmap *idmap, struct inode *dir,
                       struct dentry *dentry, umode_t mode, bool excl)
{
        handle_t *handle;
        struct inode *inode;
        int err, credits, retries = 0;

        err = dquot_initialize(dir);
        if (err)
                return err;

        credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                   EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3);
retry:
        inode = ext4_new_inode_start_handle(idmap, dir, mode, &dentry->d_name,
                                            0, NULL, EXT4_HT_DIR, credits);
        handle = ext4_journal_current_handle();
        err = PTR_ERR(inode);
        if (!IS_ERR(inode)) {
                inode->i_op = &ext4_file_inode_operations;
                inode->i_fop = &ext4_file_operations;
                ext4_set_aops(inode);
                err = ext4_add_nondir(handle, dentry, &inode);
                if (!err)
                        ext4_fc_track_create(handle, dentry);
        }
        if (handle)
                ext4_journal_stop(handle);
        if (!IS_ERR_OR_NULL(inode))
                iput(inode);
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return err;
}

static int ext4_mknod(struct mnt_idmap *idmap, struct inode *dir,
                      struct dentry *dentry, umode_t mode, dev_t rdev)
{
        handle_t *handle;
        struct inode *inode;
        int err, credits, retries = 0;

        err = dquot_initialize(dir);
        if (err)
                return err;

        credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                   EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3);
retry:
        inode = ext4_new_inode_start_handle(idmap, dir, mode, &dentry->d_name,
                                            0, NULL, EXT4_HT_DIR, credits);
        handle = ext4_journal_current_handle();
        err = PTR_ERR(inode);
        if (!IS_ERR(inode)) {
                init_special_inode(inode, inode->i_mode, rdev);
                inode->i_op = &ext4_special_inode_operations;
                err = ext4_add_nondir(handle, dentry, &inode);
                if (!err)
                        ext4_fc_track_create(handle, dentry);
        }
        if (handle)
                ext4_journal_stop(handle);
        if (!IS_ERR_OR_NULL(inode))
                iput(inode);
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return err;
}

static int ext4_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
                        struct file *file, umode_t mode)
{
        handle_t *handle;
        struct inode *inode;
        int err, retries = 0;

        err = dquot_initialize(dir);
        if (err)
                return err;

retry:
        inode = ext4_new_inode_start_handle(idmap, dir, mode,
                                            NULL, 0, NULL,
                                            EXT4_HT_DIR,
                        EXT4_MAXQUOTAS_TRANS_BLOCKS(dir->i_sb) +
                          4 + EXT4_XATTR_TRANS_BLOCKS);
        handle = ext4_journal_current_handle();
        err = PTR_ERR(inode);
        if (!IS_ERR(inode)) {
                inode->i_op = &ext4_file_inode_operations;
                inode->i_fop = &ext4_file_operations;
                ext4_set_aops(inode);
                d_tmpfile(file, inode);
                err = ext4_orphan_add(handle, inode);
                if (err)
                        goto err_unlock_inode;
                mark_inode_dirty(inode);
                unlock_new_inode(inode);
        }
        if (handle)
                ext4_journal_stop(handle);
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return finish_open_simple(file, err);
err_unlock_inode:
        ext4_journal_stop(handle);
        unlock_new_inode(inode);
        return err;
}

int ext4_init_dirblock(handle_t *handle, struct inode *inode,
                       struct buffer_head *bh, unsigned int parent_ino,
                       void *inline_buf, int inline_size)
{
        struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *) bh->b_data;
        size_t                        blocksize = bh->b_size;
        int                        csum_size = 0, header_size;

        if (ext4_has_feature_metadata_csum(inode->i_sb))
                csum_size = sizeof(struct ext4_dir_entry_tail);

        de->inode = cpu_to_le32(inode->i_ino);
        de->name_len = 1;
        de->rec_len = ext4_rec_len_to_disk(ext4_dir_rec_len(de->name_len, NULL),
                                           blocksize);
        memcpy(de->name, ".", 2);
        ext4_set_de_type(inode->i_sb, de, S_IFDIR);

        de = ext4_next_entry(de, blocksize);
        de->inode = cpu_to_le32(parent_ino);
        de->name_len = 2;
        memcpy(de->name, "..", 3);
        ext4_set_de_type(inode->i_sb, de, S_IFDIR);
        if (inline_buf) {
                de->rec_len = ext4_rec_len_to_disk(
                                        ext4_dir_rec_len(de->name_len, NULL),
                                        blocksize);
                de = ext4_next_entry(de, blocksize);
                header_size = (char *)de - bh->b_data;
                memcpy((void *)de, inline_buf, inline_size);
                ext4_update_final_de(bh->b_data, inline_size + header_size,
                        blocksize - csum_size);
        } else {
                de->rec_len = ext4_rec_len_to_disk(blocksize -
                                        (csum_size + ext4_dir_rec_len(1, NULL)),
                                        blocksize);
        }

        if (csum_size)
                ext4_initialize_dirent_tail(bh, blocksize);
        BUFFER_TRACE(dir_block, "call ext4_handle_dirty_metadata");
        set_buffer_uptodate(bh);
        set_buffer_verified(bh);
        return ext4_handle_dirty_dirblock(handle, inode, bh);
}

int ext4_init_new_dir(handle_t *handle, struct inode *dir,
                             struct inode *inode)
{
        struct buffer_head *dir_block = NULL;
        ext4_lblk_t block = 0;
        int err;

        if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
                err = ext4_try_create_inline_dir(handle, dir, inode);
                if (err < 0 && err != -ENOSPC)
                        goto out;
                if (!err)
                        goto out;
        }

        set_nlink(inode, 2);
        inode->i_size = 0;
        dir_block = ext4_append(handle, inode, &block);
        if (IS_ERR(dir_block))
                return PTR_ERR(dir_block);
        err = ext4_init_dirblock(handle, inode, dir_block, dir->i_ino, NULL, 0);
out:
        brelse(dir_block);
        return err;
}

static struct dentry *ext4_mkdir(struct mnt_idmap *idmap, struct inode *dir,
                                 struct dentry *dentry, umode_t mode)
{
        handle_t *handle;
        struct inode *inode;
        int err, err2 = 0, credits, retries = 0;

        if (EXT4_DIR_LINK_MAX(dir))
                return ERR_PTR(-EMLINK);

        err = dquot_initialize(dir);
        if (err)
                return ERR_PTR(err);

        credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                   EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3);
retry:
        inode = ext4_new_inode_start_handle(idmap, dir, S_IFDIR | mode,
                                            &dentry->d_name,
                                            0, NULL, EXT4_HT_DIR, credits);
        handle = ext4_journal_current_handle();
        err = PTR_ERR(inode);
        if (IS_ERR(inode))
                goto out_stop;

        inode->i_op = &ext4_dir_inode_operations;
        inode->i_fop = &ext4_dir_operations;
        err = ext4_init_new_dir(handle, dir, inode);
        if (err)
                goto out_clear_inode;
        err = ext4_mark_inode_dirty(handle, inode);
        if (!err)
                err = ext4_add_entry(handle, dentry, inode);
        if (err) {
out_clear_inode:
                clear_nlink(inode);
                ext4_orphan_add(handle, inode);
                unlock_new_inode(inode);
                err2 = ext4_mark_inode_dirty(handle, inode);
                if (unlikely(err2))
                        err = err2;
                ext4_journal_stop(handle);
                iput(inode);
                goto out_retry;
        }
        ext4_inc_count(dir);

        ext4_update_dx_flag(dir);
        err = ext4_mark_inode_dirty(handle, dir);
        if (err)
                goto out_clear_inode;
        d_instantiate_new(dentry, inode);
        ext4_fc_track_create(handle, dentry);
        if (IS_DIRSYNC(dir))
                ext4_handle_sync(handle);

out_stop:
        if (handle)
                ext4_journal_stop(handle);
out_retry:
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return ERR_PTR(err);
}

/*
 * routine to check that the specified directory is empty (for rmdir)
 */
bool ext4_empty_dir(struct inode *inode)
{
        unsigned int offset;
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de;
        struct super_block *sb;

        if (ext4_has_inline_data(inode)) {
                int has_inline_data = 1;
                int ret;

                ret = empty_inline_dir(inode, &has_inline_data);
                if (has_inline_data)
                        return ret;
        }

        sb = inode->i_sb;
        if (inode->i_size < ext4_dir_rec_len(1, NULL) +
                                        ext4_dir_rec_len(2, NULL)) {
                EXT4_ERROR_INODE(inode, "invalid size");
                return false;
        }
        bh = ext4_read_dirblock(inode, 0, EITHER);
        if (IS_ERR(bh))
                return false;

        de = (struct ext4_dir_entry_2 *) bh->b_data;
        if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size,
                                 0) ||
            le32_to_cpu(de->inode) != inode->i_ino || de->name_len != 1 ||
            de->name[0] != '.') {
                ext4_warning_inode(inode, "directory missing '.'");
                brelse(bh);
                return false;
        }
        offset = ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize);
        de = ext4_next_entry(de, sb->s_blocksize);
        if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size,
                                 offset) ||
            le32_to_cpu(de->inode) == 0 || de->name_len != 2 ||
            de->name[0] != '.' || de->name[1] != '.') {
                ext4_warning_inode(inode, "directory missing '..'");
                brelse(bh);
                return false;
        }
        offset += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize);
        while (offset < inode->i_size) {
                if (!(offset & (sb->s_blocksize - 1))) {
                        unsigned int lblock;
                        brelse(bh);
                        lblock = offset >> EXT4_BLOCK_SIZE_BITS(sb);
                        bh = ext4_read_dirblock(inode, lblock, EITHER);
                        if (bh == NULL) {
                                offset += sb->s_blocksize;
                                continue;
                        }
                        if (IS_ERR(bh))
                                return false;
                }
                de = (struct ext4_dir_entry_2 *) (bh->b_data +
                                        (offset & (sb->s_blocksize - 1)));
                if (ext4_check_dir_entry(inode, NULL, de, bh,
                                         bh->b_data, bh->b_size, offset) ||
                    le32_to_cpu(de->inode)) {
                        brelse(bh);
                        return false;
                }
                offset += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize);
        }
        brelse(bh);
        return true;
}

static int ext4_rmdir(struct inode *dir, struct dentry *dentry)
{
        int retval;
        struct inode *inode;
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de = NULL;
        handle_t *handle = NULL;

        retval = ext4_emergency_state(dir->i_sb);
        if (unlikely(retval))
                return retval;

        /* Initialize quotas before so that eventual writes go in
         * separate transaction */
        retval = dquot_initialize(dir);
        if (retval)
                return retval;
        retval = dquot_initialize(d_inode(dentry));
        if (retval)
                return retval;

        retval = -ENOENT;
        bh = ext4_find_entry(dir, &dentry->d_name, &de, NULL);
        if (IS_ERR(bh))
                return PTR_ERR(bh);
        if (!bh)
                goto end_rmdir;

        inode = d_inode(dentry);

        retval = -EFSCORRUPTED;
        if (le32_to_cpu(de->inode) != inode->i_ino)
                goto end_rmdir;

        retval = -ENOTEMPTY;
        if (!ext4_empty_dir(inode))
                goto end_rmdir;

        handle = ext4_journal_start(dir, EXT4_HT_DIR,
                                    EXT4_DATA_TRANS_BLOCKS(dir->i_sb));
        if (IS_ERR(handle)) {
                retval = PTR_ERR(handle);
                handle = NULL;
                goto end_rmdir;
        }

        if (IS_DIRSYNC(dir))
                ext4_handle_sync(handle);

        retval = ext4_delete_entry(handle, dir, de, bh);
        if (retval)
                goto end_rmdir;
        if (!EXT4_DIR_LINK_EMPTY(inode))
                ext4_warning_inode(inode,
                             "empty directory '%.*s' has too many links (%u)",
                             dentry->d_name.len, dentry->d_name.name,
                             inode->i_nlink);
        inode_inc_iversion(inode);
        clear_nlink(inode);
        /* There's no need to set i_disksize: the fact that i_nlink is
         * zero will ensure that the right thing happens during any
         * recovery. */
        inode->i_size = 0;
        ext4_orphan_add(handle, inode);
        inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
        inode_set_ctime_current(inode);
        retval = ext4_mark_inode_dirty(handle, inode);
        if (retval)
                goto end_rmdir;
        ext4_dec_count(dir);
        ext4_update_dx_flag(dir);
        ext4_fc_track_unlink(handle, dentry);
        retval = ext4_mark_inode_dirty(handle, dir);

        /* VFS negative dentries are incompatible with Encoding and
         * Case-insensitiveness. Eventually we'll want avoid
         * invalidating the dentries here, alongside with returning the
         * negative dentries at ext4_lookup(), when it is better
         * supported by the VFS for the CI case.
         */
        if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
                d_invalidate(dentry);

end_rmdir:
        brelse(bh);
        if (handle)
                ext4_journal_stop(handle);
        return retval;
}

int __ext4_unlink(struct inode *dir, const struct qstr *d_name,
                  struct inode *inode,
                  struct dentry *dentry /* NULL during fast_commit recovery */)
{
        int retval = -ENOENT;
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de = NULL;
        handle_t *handle;
        int skip_remove_dentry = 0;

        /*
         * Keep this outside the transaction; it may have to set up the
         * directory's encryption key, which isn't GFP_NOFS-safe.
         */
        bh = ext4_find_entry(dir, d_name, &de, NULL);
        if (IS_ERR(bh))
                return PTR_ERR(bh);

        if (!bh)
                return -ENOENT;

        if (le32_to_cpu(de->inode) != inode->i_ino) {
                /*
                 * It's okay if we find dont find dentry which matches
                 * the inode. That's because it might have gotten
                 * renamed to a different inode number
                 */
                if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
                        skip_remove_dentry = 1;
                else
                        goto out_bh;
        }

        handle = ext4_journal_start(dir, EXT4_HT_DIR,
                                    EXT4_DATA_TRANS_BLOCKS(dir->i_sb));
        if (IS_ERR(handle)) {
                retval = PTR_ERR(handle);
                goto out_bh;
        }

        if (IS_DIRSYNC(dir))
                ext4_handle_sync(handle);

        if (!skip_remove_dentry) {
                retval = ext4_delete_entry(handle, dir, de, bh);
                if (retval)
                        goto out_handle;
                inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
                ext4_update_dx_flag(dir);
                retval = ext4_mark_inode_dirty(handle, dir);
                if (retval)
                        goto out_handle;
        } else {
                retval = 0;
        }
        if (inode->i_nlink == 0)
                ext4_warning_inode(inode, "Deleting file '%.*s' with no links",
                                   d_name->len, d_name->name);
        else
                drop_nlink(inode);
        if (!inode->i_nlink)
                ext4_orphan_add(handle, inode);
        inode_set_ctime_current(inode);
        retval = ext4_mark_inode_dirty(handle, inode);
        if (dentry && !retval)
                ext4_fc_track_unlink(handle, dentry);
out_handle:
        ext4_journal_stop(handle);
out_bh:
        brelse(bh);
        return retval;
}

static int ext4_unlink(struct inode *dir, struct dentry *dentry)
{
        int retval;

        retval = ext4_emergency_state(dir->i_sb);
        if (unlikely(retval))
                return retval;

        trace_ext4_unlink_enter(dir, dentry);
        /*
         * Initialize quotas before so that eventual writes go
         * in separate transaction
         */
        retval = dquot_initialize(dir);
        if (retval)
                goto out_trace;
        retval = dquot_initialize(d_inode(dentry));
        if (retval)
                goto out_trace;

        retval = __ext4_unlink(dir, &dentry->d_name, d_inode(dentry), dentry);

        /* VFS negative dentries are incompatible with Encoding and
         * Case-insensitiveness. Eventually we'll want avoid
         * invalidating the dentries here, alongside with returning the
         * negative dentries at ext4_lookup(), when it is  better
         * supported by the VFS for the CI case.
         */
        if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir))
                d_invalidate(dentry);

out_trace:
        trace_ext4_unlink_exit(dentry, retval);
        return retval;
}

static int ext4_init_symlink_block(handle_t *handle, struct inode *inode,
                                   struct fscrypt_str *disk_link)
{
        struct buffer_head *bh;
        char *kaddr;
        int err = 0;

        bh = ext4_bread(handle, inode, 0, EXT4_GET_BLOCKS_CREATE);
        if (IS_ERR(bh))
                return PTR_ERR(bh);

        BUFFER_TRACE(bh, "get_write_access");
        err = ext4_journal_get_write_access(handle, inode->i_sb, bh, EXT4_JTR_NONE);
        if (err)
                goto out;

        kaddr = (char *)bh->b_data;
        memcpy(kaddr, disk_link->name, disk_link->len);
        inode->i_size = disk_link->len - 1;
        EXT4_I(inode)->i_disksize = inode->i_size;
        err = ext4_handle_dirty_metadata(handle, inode, bh);
out:
        brelse(bh);
        return err;
}

static int ext4_symlink(struct mnt_idmap *idmap, struct inode *dir,
                        struct dentry *dentry, const char *symname)
{
        handle_t *handle;
        struct inode *inode;
        int err, len = strlen(symname);
        int credits;
        struct fscrypt_str disk_link;
        int retries = 0;

        err = ext4_emergency_state(dir->i_sb);
        if (unlikely(err))
                return err;

        err = fscrypt_prepare_symlink(dir, symname, len, dir->i_sb->s_blocksize,
                                      &disk_link);
        if (err)
                return err;

        err = dquot_initialize(dir);
        if (err)
                return err;

        /*
         * EXT4_INDEX_EXTRA_TRANS_BLOCKS for addition of entry into the
         * directory. +3 for inode, inode bitmap, group descriptor allocation.
         * EXT4_DATA_TRANS_BLOCKS for the data block allocation and
         * modification.
         */
        credits = EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                  EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3;
retry:
        inode = ext4_new_inode_start_handle(idmap, dir, S_IFLNK|S_IRWXUGO,
                                            &dentry->d_name, 0, NULL,
                                            EXT4_HT_DIR, credits);
        handle = ext4_journal_current_handle();
        if (IS_ERR(inode)) {
                if (handle)
                        ext4_journal_stop(handle);
                err = PTR_ERR(inode);
                goto out_retry;
        }

        if (IS_ENCRYPTED(inode)) {
                err = fscrypt_encrypt_symlink(inode, symname, len, &disk_link);
                if (err)
                        goto err_drop_inode;
                inode->i_op = &ext4_encrypted_symlink_inode_operations;
        } else {
                if ((disk_link.len > EXT4_N_BLOCKS * 4)) {
                        inode->i_op = &ext4_symlink_inode_operations;
                } else {
                        inode->i_op = &ext4_fast_symlink_inode_operations;
                }
        }

        if ((disk_link.len > EXT4_N_BLOCKS * 4)) {
                /* alloc symlink block and fill it */
                err = ext4_init_symlink_block(handle, inode, &disk_link);
                if (err)
                        goto err_drop_inode;
        } else {
                /* clear the extent format for fast symlink */
                ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS);
                memcpy((char *)&EXT4_I(inode)->i_data, disk_link.name,
                       disk_link.len);
                inode->i_size = disk_link.len - 1;
                EXT4_I(inode)->i_disksize = inode->i_size;
                if (!IS_ENCRYPTED(inode))
                        inode_set_cached_link(inode, (char *)&EXT4_I(inode)->i_data,
                                              inode->i_size);
        }
        err = ext4_add_nondir(handle, dentry, &inode);
        if (handle)
                ext4_journal_stop(handle);
        iput(inode);
        goto out_retry;

err_drop_inode:
        clear_nlink(inode);
        ext4_mark_inode_dirty(handle, inode);
        ext4_orphan_add(handle, inode);
        unlock_new_inode(inode);
        if (handle)
                ext4_journal_stop(handle);
        iput(inode);
out_retry:
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        if (disk_link.name != (unsigned char *)symname)
                kfree(disk_link.name);
        return err;
}

int __ext4_link(struct inode *dir, struct inode *inode,
                const struct qstr *d_name, struct dentry *dentry)
{
        handle_t *handle;
        int err, retries = 0;
retry:
        handle = ext4_journal_start(dir, EXT4_HT_DIR,
                (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) +
                 EXT4_INDEX_EXTRA_TRANS_BLOCKS) + 1);
        if (IS_ERR(handle))
                return PTR_ERR(handle);

        if (IS_DIRSYNC(dir))
                ext4_handle_sync(handle);

        inode_set_ctime_current(inode);
        ext4_inc_count(inode);

        err = __ext4_add_entry(handle, dir, d_name, inode);
        if (!err) {
                err = ext4_mark_inode_dirty(handle, inode);
                /* this can happen only for tmpfile being
                 * linked the first time
                 */
                if (inode->i_nlink == 1)
                        ext4_orphan_del(handle, inode);
                if (dentry)
                        ext4_fc_track_link(handle, inode, dentry);
        } else {
                drop_nlink(inode);
        }
        ext4_journal_stop(handle);
        if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries))
                goto retry;
        return err;
}

static int ext4_link(struct dentry *old_dentry,
                     struct inode *dir, struct dentry *dentry)
{
        struct inode *inode = d_inode(old_dentry);
        int err;

        if (inode->i_nlink >= EXT4_LINK_MAX)
                return -EMLINK;

        err = fscrypt_prepare_link(old_dentry, dir, dentry);
        if (err)
                return err;

        if ((ext4_test_inode_flag(dir, EXT4_INODE_PROJINHERIT)) &&
            (!projid_eq(EXT4_I(dir)->i_projid,
                        EXT4_I(old_dentry->d_inode)->i_projid)))
                return -EXDEV;

        err = dquot_initialize(dir);
        if (err)
                return err;
        err = __ext4_link(dir, inode, &dentry->d_name, dentry);
        if (!err) {
                ihold(inode);
                d_instantiate(dentry, inode);
        }
        return err;
}
/*
 * Try to find buffer head where contains the parent block.
 * It should be the inode block if it is inlined or the 1st block
 * if it is a normal dir.
 */
static struct buffer_head *ext4_get_first_dir_block(handle_t *handle,
                                        struct inode *inode,
                                        int *retval,
                                        struct ext4_dir_entry_2 **parent_de,
                                        int *inlined)
{
        struct buffer_head *bh;

        if (!ext4_has_inline_data(inode)) {
                struct ext4_dir_entry_2 *de;
                unsigned int offset;

                bh = ext4_read_dirblock(inode, 0, EITHER);
                if (IS_ERR(bh)) {
                        *retval = PTR_ERR(bh);
                        return NULL;
                }

                de = (struct ext4_dir_entry_2 *) bh->b_data;
                if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data,
                                         bh->b_size, 0) ||
                    le32_to_cpu(de->inode) != inode->i_ino ||
                    de->name_len != 1 || de->name[0] != '.') {
                        EXT4_ERROR_INODE(inode, "directory missing '.'");
                        brelse(bh);
                        *retval = -EFSCORRUPTED;
                        return NULL;
                }
                offset = ext4_rec_len_from_disk(de->rec_len,
                                                inode->i_sb->s_blocksize);
                de = ext4_next_entry(de, inode->i_sb->s_blocksize);
                if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data,
                                         bh->b_size, offset) ||
                    le32_to_cpu(de->inode) == 0 || de->name_len != 2 ||
                    de->name[0] != '.' || de->name[1] != '.') {
                        EXT4_ERROR_INODE(inode, "directory missing '..'");
                        brelse(bh);
                        *retval = -EFSCORRUPTED;
                        return NULL;
                }
                *parent_de = de;

                return bh;
        }

        *inlined = 1;
        return ext4_get_first_inline_block(inode, parent_de, retval);
}

struct ext4_renament {
        struct inode *dir;
        struct dentry *dentry;
        struct inode *inode;
        bool is_dir;
        int dir_nlink_delta;

        /* entry for "dentry" */
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de;
        int inlined;

        /* entry for ".." in inode if it's a directory */
        struct buffer_head *dir_bh;
        struct ext4_dir_entry_2 *parent_de;
        int dir_inlined;
};

static int ext4_rename_dir_prepare(handle_t *handle, struct ext4_renament *ent, bool is_cross)
{
        int retval;

        ent->is_dir = true;
        if (!is_cross)
                return 0;

        ent->dir_bh = ext4_get_first_dir_block(handle, ent->inode,
                                              &retval, &ent->parent_de,
                                              &ent->dir_inlined);
        if (!ent->dir_bh)
                return retval;
        if (le32_to_cpu(ent->parent_de->inode) != ent->dir->i_ino)
                return -EFSCORRUPTED;
        BUFFER_TRACE(ent->dir_bh, "get_write_access");
        return ext4_journal_get_write_access(handle, ent->dir->i_sb,
                                             ent->dir_bh, EXT4_JTR_NONE);
}

static int ext4_rename_dir_finish(handle_t *handle, struct ext4_renament *ent,
                                  unsigned dir_ino)
{
        int retval;

        if (!ent->dir_bh)
                return 0;

        ent->parent_de->inode = cpu_to_le32(dir_ino);
        BUFFER_TRACE(ent->dir_bh, "call ext4_handle_dirty_metadata");
        if (!ent->dir_inlined) {
                if (is_dx(ent->inode)) {
                        retval = ext4_handle_dirty_dx_node(handle,
                                                           ent->inode,
                                                           ent->dir_bh);
                } else {
                        retval = ext4_handle_dirty_dirblock(handle, ent->inode,
                                                            ent->dir_bh);
                }
        } else {
                retval = ext4_mark_inode_dirty(handle, ent->inode);
        }
        if (retval) {
                ext4_std_error(ent->dir->i_sb, retval);
                return retval;
        }
        return 0;
}

static int ext4_setent(handle_t *handle, struct ext4_renament *ent,
                       unsigned ino, unsigned file_type)
{
        int retval, retval2;

        BUFFER_TRACE(ent->bh, "get write access");
        retval = ext4_journal_get_write_access(handle, ent->dir->i_sb, ent->bh,
                                               EXT4_JTR_NONE);
        if (retval)
                return retval;
        ent->de->inode = cpu_to_le32(ino);
        if (ext4_has_feature_filetype(ent->dir->i_sb))
                ent->de->file_type = file_type;
        inode_inc_iversion(ent->dir);
        inode_set_mtime_to_ts(ent->dir, inode_set_ctime_current(ent->dir));
        retval = ext4_mark_inode_dirty(handle, ent->dir);
        BUFFER_TRACE(ent->bh, "call ext4_handle_dirty_metadata");
        if (!ent->inlined) {
                retval2 = ext4_handle_dirty_dirblock(handle, ent->dir, ent->bh);
                if (unlikely(retval2)) {
                        ext4_std_error(ent->dir->i_sb, retval2);
                        return retval2;
                }
        }
        return retval;
}

static void ext4_resetent(handle_t *handle, struct ext4_renament *ent,
                          unsigned ino, unsigned file_type)
{
        struct ext4_renament old = *ent;
        int retval = 0;

        /*
         * old->de could have moved from under us during make indexed dir,
         * so the old->de may no longer valid and need to find it again
         * before reset old inode info.
         */
        old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de,
                                 &old.inlined);
        if (IS_ERR(old.bh))
                retval = PTR_ERR(old.bh);
        if (!old.bh)
                retval = -ENOENT;
        if (retval) {
                ext4_std_error(old.dir->i_sb, retval);
                return;
        }

        ext4_setent(handle, &old, ino, file_type);
        brelse(old.bh);
}

static int ext4_find_delete_entry(handle_t *handle, struct inode *dir,
                                  const struct qstr *d_name)
{
        int retval = -ENOENT;
        struct buffer_head *bh;
        struct ext4_dir_entry_2 *de = NULL;

        bh = ext4_find_entry(dir, d_name, &de, NULL);
        if (IS_ERR(bh))
                return PTR_ERR(bh);
        if (bh) {
                retval = ext4_delete_entry(handle, dir, de, bh);
                brelse(bh);
        }
        return retval;
}

static void ext4_rename_delete(handle_t *handle, struct ext4_renament *ent,
                               int force_reread)
{
        int retval;
        /*
         * ent->de could have moved from under us during htree split, so make
         * sure that we are deleting the right entry.  We might also be pointing
         * to a stale entry in the unused part of ent->bh so just checking inum
         * and the name isn't enough.
         */
        if (le32_to_cpu(ent->de->inode) != ent->inode->i_ino ||
            ent->de->name_len != ent->dentry->d_name.len ||
            strncmp(ent->de->name, ent->dentry->d_name.name,
                    ent->de->name_len) ||
            force_reread) {
                retval = ext4_find_delete_entry(handle, ent->dir,
                                                &ent->dentry->d_name);
        } else {
                retval = ext4_delete_entry(handle, ent->dir, ent->de, ent->bh);
                if (retval == -ENOENT) {
                        retval = ext4_find_delete_entry(handle, ent->dir,
                                                        &ent->dentry->d_name);
                }
        }

        if (retval) {
                ext4_warning_inode(ent->dir,
                                   "Deleting old file: nlink %d, error=%d",
                                   ent->dir->i_nlink, retval);
        }
}

static void ext4_update_dir_count(handle_t *handle, struct ext4_renament *ent)
{
        if (ent->dir_nlink_delta) {
                if (ent->dir_nlink_delta == -1)
                        ext4_dec_count(ent->dir);
                else
                        ext4_inc_count(ent->dir);
                ext4_mark_inode_dirty(handle, ent->dir);
        }
}

static struct inode *ext4_whiteout_for_rename(struct mnt_idmap *idmap,
                                              struct ext4_renament *ent,
                                              int credits, handle_t **h)
{
        struct inode *wh;
        handle_t *handle;
        int retries = 0;

        /*
         * for inode block, sb block, group summaries,
         * and inode bitmap
         */
        credits += (EXT4_MAXQUOTAS_TRANS_BLOCKS(ent->dir->i_sb) +
                    EXT4_XATTR_TRANS_BLOCKS + 4);
retry:
        wh = ext4_new_inode_start_handle(idmap, ent->dir,
                                         S_IFCHR | WHITEOUT_MODE,
                                         &ent->dentry->d_name, 0, NULL,
                                         EXT4_HT_DIR, credits);

        handle = ext4_journal_current_handle();
        if (IS_ERR(wh)) {
                if (handle)
                        ext4_journal_stop(handle);
                if (PTR_ERR(wh) == -ENOSPC &&
                    ext4_should_retry_alloc(ent->dir->i_sb, &retries))
                        goto retry;
        } else {
                *h = handle;
                init_special_inode(wh, wh->i_mode, WHITEOUT_DEV);
                wh->i_op = &ext4_special_inode_operations;
        }
        return wh;
}

/*
 * Anybody can rename anything with this: the permission checks are left to the
 * higher-level routines.
 *
 * n.b.  old_{dentry,inode) refers to the source dentry/inode
 * while new_{dentry,inode) refers to the destination dentry/inode
 * This comes from rename(const char *oldpath, const char *newpath)
 */
static int ext4_rename(struct mnt_idmap *idmap, struct inode *old_dir,
                       struct dentry *old_dentry, struct inode *new_dir,
                       struct dentry *new_dentry, unsigned int flags)
{
        handle_t *handle = NULL;
        struct ext4_renament old = {
                .dir = old_dir,
                .dentry = old_dentry,
                .inode = d_inode(old_dentry),
        };
        struct ext4_renament new = {
                .dir = new_dir,
                .dentry = new_dentry,
                .inode = d_inode(new_dentry),
        };
        int force_reread;
        int retval;
        struct inode *whiteout = NULL;
        int credits;
        u8 old_file_type;

        if (new.inode && new.inode->i_nlink == 0) {
                EXT4_ERROR_INODE(new.inode,
                                 "target of rename is already freed");
                return -EFSCORRUPTED;
        }

        if ((ext4_test_inode_flag(new_dir, EXT4_INODE_PROJINHERIT)) &&
            (!projid_eq(EXT4_I(new_dir)->i_projid,
                        EXT4_I(old_dentry->d_inode)->i_projid)))
                return -EXDEV;

        retval = dquot_initialize(old.dir);
        if (retval)
                return retval;
        retval = dquot_initialize(old.inode);
        if (retval)
                return retval;
        retval = dquot_initialize(new.dir);
        if (retval)
                return retval;

        /* Initialize quotas before so that eventual writes go
         * in separate transaction */
        if (new.inode) {
                retval = dquot_initialize(new.inode);
                if (retval)
                        return retval;
        }

        old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de,
                                 &old.inlined);
        if (IS_ERR(old.bh))
                return PTR_ERR(old.bh);

        /*
         *  Check for inode number is _not_ due to possible IO errors.
         *  We might rmdir the source, keep it as pwd of some process
         *  and merrily kill the link to whatever was created under the
         *  same name. Goodbye sticky bit ;-<
         */
        retval = -ENOENT;
        if (!old.bh || le32_to_cpu(old.de->inode) != old.inode->i_ino)
                goto release_bh;

        new.bh = ext4_find_entry(new.dir, &new.dentry->d_name,
                                 &new.de, &new.inlined);
        if (IS_ERR(new.bh)) {
                retval = PTR_ERR(new.bh);
                new.bh = NULL;
                goto release_bh;
        }
        if (new.bh) {
                if (!new.inode) {
                        brelse(new.bh);
                        new.bh = NULL;
                }
        }
        if (new.inode && !test_opt(new.dir->i_sb, NO_AUTO_DA_ALLOC))
                ext4_alloc_da_blocks(old.inode);

        credits = (2 * EXT4_DATA_TRANS_BLOCKS(old.dir->i_sb) +
                   EXT4_INDEX_EXTRA_TRANS_BLOCKS + 2);
        if (!(flags & RENAME_WHITEOUT)) {
                handle = ext4_journal_start(old.dir, EXT4_HT_DIR, credits);
                if (IS_ERR(handle)) {
                        retval = PTR_ERR(handle);
                        goto release_bh;
                }
        } else {
                whiteout = ext4_whiteout_for_rename(idmap, &old, credits, &handle);
                if (IS_ERR(whiteout)) {
                        retval = PTR_ERR(whiteout);
                        goto release_bh;
                }
        }

        old_file_type = old.de->file_type;
        if (IS_DIRSYNC(old.dir) || IS_DIRSYNC(new.dir))
                ext4_handle_sync(handle);

        if (S_ISDIR(old.inode->i_mode)) {
                if (new.inode) {
                        retval = -ENOTEMPTY;
                        if (!ext4_empty_dir(new.inode))
                                goto end_rename;
                } else {
                        retval = -EMLINK;
                        if (new.dir != old.dir && EXT4_DIR_LINK_MAX(new.dir))
                                goto end_rename;
                }
                retval = ext4_rename_dir_prepare(handle, &old, new.dir != old.dir);
                if (retval)
                        goto end_rename;
        }
        /*
         * If we're renaming a file within an inline_data dir and adding or
         * setting the new dirent causes a conversion from inline_data to
         * extents/blockmap, we need to force the dirent delete code to
         * re-read the directory, or else we end up trying to delete a dirent
         * from what is now the extent tree root (or a block map).
         */
        force_reread = (new.dir->i_ino == old.dir->i_ino &&
                        ext4_test_inode_flag(new.dir, EXT4_INODE_INLINE_DATA));

        if (whiteout) {
                /*
                 * Do this before adding a new entry, so the old entry is sure
                 * to be still pointing to the valid old entry.
                 */
                retval = ext4_setent(handle, &old, whiteout->i_ino,
                                     EXT4_FT_CHRDEV);
                if (retval)
                        goto end_rename;
                retval = ext4_mark_inode_dirty(handle, whiteout);
                if (unlikely(retval))
                        goto end_rename;

        }
        if (!new.bh) {
                retval = ext4_add_entry(handle, new.dentry, old.inode);
                if (retval)
                        goto end_rename;
        } else {
                retval = ext4_setent(handle, &new,
                                     old.inode->i_ino, old_file_type);
                if (retval)
                        goto end_rename;
        }
        if (force_reread)
                force_reread = !ext4_test_inode_flag(new.dir,
                                                     EXT4_INODE_INLINE_DATA);

        /*
         * Like most other Unix systems, set the ctime for inodes on a
         * rename.
         */
        inode_set_ctime_current(old.inode);
        retval = ext4_mark_inode_dirty(handle, old.inode);
        if (unlikely(retval))
                goto end_rename;

        if (!whiteout) {
                /*
                 * ok, that's it
                 */
                ext4_rename_delete(handle, &old, force_reread);
        }

        if (new.inode) {
                ext4_dec_count(new.inode);
                inode_set_ctime_current(new.inode);
        }
        inode_set_mtime_to_ts(old.dir, inode_set_ctime_current(old.dir));
        ext4_update_dx_flag(old.dir);
        if (old.is_dir) {
                retval = ext4_rename_dir_finish(handle, &old, new.dir->i_ino);
                if (retval)
                        goto end_rename;

                ext4_dec_count(old.dir);
                if (new.inode) {
                        /* checked ext4_empty_dir above, can't have another
                         * parent, ext4_dec_count() won't work for many-linked
                         * dirs */
                        clear_nlink(new.inode);
                } else {
                        ext4_inc_count(new.dir);
                        ext4_update_dx_flag(new.dir);
                        retval = ext4_mark_inode_dirty(handle, new.dir);
                        if (unlikely(retval))
                                goto end_rename;
                }
        }
        retval = ext4_mark_inode_dirty(handle, old.dir);
        if (unlikely(retval))
                goto end_rename;

        if (old.is_dir) {
                /*
                 * We disable fast commits here that's because the
                 * replay code is not yet capable of changing dot dot
                 * dirents in directories.
                 */
                ext4_fc_mark_ineligible(old.inode->i_sb,
                        EXT4_FC_REASON_RENAME_DIR, handle);
        } else {
                struct super_block *sb = old.inode->i_sb;

                if (new.inode)
                        ext4_fc_track_unlink(handle, new.dentry);
                if (test_opt2(sb, JOURNAL_FAST_COMMIT) &&
                    !(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
                    !(ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE))) {
                        __ext4_fc_track_link(handle, old.inode, new.dentry);
                        __ext4_fc_track_unlink(handle, old.inode, old.dentry);
                        if (whiteout)
                                __ext4_fc_track_create(handle, whiteout,
                                                       old.dentry);
                }
        }

        if (new.inode) {
                retval = ext4_mark_inode_dirty(handle, new.inode);
                if (unlikely(retval))
                        goto end_rename;
                if (!new.inode->i_nlink)
                        ext4_orphan_add(handle, new.inode);
        }
        retval = 0;

end_rename:
        if (whiteout) {
                if (retval) {
                        ext4_resetent(handle, &old,
                                      old.inode->i_ino, old_file_type);
                        drop_nlink(whiteout);
                        ext4_mark_inode_dirty(handle, whiteout);
                        ext4_orphan_add(handle, whiteout);
                }
                unlock_new_inode(whiteout);
                ext4_journal_stop(handle);
                iput(whiteout);
        } else {
                ext4_journal_stop(handle);
        }
release_bh:
        brelse(old.dir_bh);
        brelse(old.bh);
        brelse(new.bh);

        return retval;
}

static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
                             struct inode *new_dir, struct dentry *new_dentry)
{
        handle_t *handle = NULL;
        struct ext4_renament old = {
                .dir = old_dir,
                .dentry = old_dentry,
                .inode = d_inode(old_dentry),
        };
        struct ext4_renament new = {
                .dir = new_dir,
                .dentry = new_dentry,
                .inode = d_inode(new_dentry),
        };
        u8 new_file_type;
        int retval;

        if ((ext4_test_inode_flag(new_dir, EXT4_INODE_PROJINHERIT) &&
             !projid_eq(EXT4_I(new_dir)->i_projid,
                        EXT4_I(old_dentry->d_inode)->i_projid)) ||
            (ext4_test_inode_flag(old_dir, EXT4_INODE_PROJINHERIT) &&
             !projid_eq(EXT4_I(old_dir)->i_projid,
                        EXT4_I(new_dentry->d_inode)->i_projid)))
                return -EXDEV;

        retval = dquot_initialize(old.dir);
        if (retval)
                return retval;
        retval = dquot_initialize(new.dir);
        if (retval)
                return retval;

        old.bh = ext4_find_entry(old.dir, &old.dentry->d_name,
                                 &old.de, &old.inlined);
        if (IS_ERR(old.bh))
                return PTR_ERR(old.bh);
        /*
         *  Check for inode number is _not_ due to possible IO errors.
         *  We might rmdir the source, keep it as pwd of some process
         *  and merrily kill the link to whatever was created under the
         *  same name. Goodbye sticky bit ;-<
         */
        retval = -ENOENT;
        if (!old.bh || le32_to_cpu(old.de->inode) != old.inode->i_ino)
                goto end_rename;

        new.bh = ext4_find_entry(new.dir, &new.dentry->d_name,
                                 &new.de, &new.inlined);
        if (IS_ERR(new.bh)) {
                retval = PTR_ERR(new.bh);
                new.bh = NULL;
                goto end_rename;
        }

        /* RENAME_EXCHANGE case: old *and* new must both exist */
        if (!new.bh || le32_to_cpu(new.de->inode) != new.inode->i_ino)
                goto end_rename;

        handle = ext4_journal_start(old.dir, EXT4_HT_DIR,
                (2 * EXT4_DATA_TRANS_BLOCKS(old.dir->i_sb) +
                 2 * EXT4_INDEX_EXTRA_TRANS_BLOCKS + 2));
        if (IS_ERR(handle)) {
                retval = PTR_ERR(handle);
                handle = NULL;
                goto end_rename;
        }

        if (IS_DIRSYNC(old.dir) || IS_DIRSYNC(new.dir))
                ext4_handle_sync(handle);

        if (S_ISDIR(old.inode->i_mode)) {
                retval = ext4_rename_dir_prepare(handle, &old, new.dir != old.dir);
                if (retval)
                        goto end_rename;
        }
        if (S_ISDIR(new.inode->i_mode)) {
                retval = ext4_rename_dir_prepare(handle, &new, new.dir != old.dir);
                if (retval)
                        goto end_rename;
        }

        /*
         * Other than the special case of overwriting a directory, parents'
         * nlink only needs to be modified if this is a cross directory rename.
         */
        if (old.dir != new.dir && old.is_dir != new.is_dir) {
                old.dir_nlink_delta = old.is_dir ? -1 : 1;
                new.dir_nlink_delta = -old.dir_nlink_delta;
                retval = -EMLINK;
                if ((old.dir_nlink_delta > 0 && EXT4_DIR_LINK_MAX(old.dir)) ||
                    (new.dir_nlink_delta > 0 && EXT4_DIR_LINK_MAX(new.dir)))
                        goto end_rename;
        }

        new_file_type = new.de->file_type;
        retval = ext4_setent(handle, &new, old.inode->i_ino, old.de->file_type);
        if (retval)
                goto end_rename;

        retval = ext4_setent(handle, &old, new.inode->i_ino, new_file_type);
        if (retval)
                goto end_rename;

        /*
         * Like most other Unix systems, set the ctime for inodes on a
         * rename.
         */
        inode_set_ctime_current(old.inode);
        inode_set_ctime_current(new.inode);
        retval = ext4_mark_inode_dirty(handle, old.inode);
        if (unlikely(retval))
                goto end_rename;
        retval = ext4_mark_inode_dirty(handle, new.inode);
        if (unlikely(retval))
                goto end_rename;
        ext4_fc_mark_ineligible(new.inode->i_sb,
                                EXT4_FC_REASON_CROSS_RENAME, handle);
        if (old.dir_bh) {
                retval = ext4_rename_dir_finish(handle, &old, new.dir->i_ino);
                if (retval)
                        goto end_rename;
        }
        if (new.dir_bh) {
                retval = ext4_rename_dir_finish(handle, &new, old.dir->i_ino);
                if (retval)
                        goto end_rename;
        }
        ext4_update_dir_count(handle, &old);
        ext4_update_dir_count(handle, &new);
        retval = 0;

end_rename:
        brelse(old.dir_bh);
        brelse(new.dir_bh);
        brelse(old.bh);
        brelse(new.bh);
        if (handle)
                ext4_journal_stop(handle);
        return retval;
}

static int ext4_rename2(struct mnt_idmap *idmap,
                        struct inode *old_dir, struct dentry *old_dentry,
                        struct inode *new_dir, struct dentry *new_dentry,
                        unsigned int flags)
{
        int err;

        err = ext4_emergency_state(old_dir->i_sb);
        if (unlikely(err))
                return err;

        if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
                return -EINVAL;

        err = fscrypt_prepare_rename(old_dir, old_dentry, new_dir, new_dentry,
                                     flags);
        if (err)
                return err;

        if (flags & RENAME_EXCHANGE) {
                return ext4_cross_rename(old_dir, old_dentry,
                                         new_dir, new_dentry);
        }

        return ext4_rename(idmap, old_dir, old_dentry, new_dir, new_dentry, flags);
}

/*
 * directories can handle most operations...
 */
const struct inode_operations ext4_dir_inode_operations = {
        .create                = ext4_create,
        .lookup                = ext4_lookup,
        .link                = ext4_link,
        .unlink                = ext4_unlink,
        .symlink        = ext4_symlink,
        .mkdir                = ext4_mkdir,
        .rmdir                = ext4_rmdir,
        .mknod                = ext4_mknod,
        .tmpfile        = ext4_tmpfile,
        .rename                = ext4_rename2,
        .setattr        = ext4_setattr,
        .getattr        = ext4_getattr,
        .listxattr        = ext4_listxattr,
        .get_inode_acl        = ext4_get_acl,
        .set_acl        = ext4_set_acl,
        .fiemap         = ext4_fiemap,
        .fileattr_get        = ext4_fileattr_get,
        .fileattr_set        = ext4_fileattr_set,
};

const struct inode_operations ext4_special_inode_operations = {
        .setattr        = ext4_setattr,
        .getattr        = ext4_getattr,
        .listxattr        = ext4_listxattr,
        .get_inode_acl        = ext4_get_acl,
        .set_acl        = ext4_set_acl,
};
















    7 






























   10 





















    4 
    7 


   10 



   10 













   12 

















    7 




   12 



   12 



   12 











   12 






   10 















   12 


   10 

























   13 









    7 







    1 


    5 


   12 



   13 


   13 








   15 



    1 


    1 
   13 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/module.h>

#include <net/sock.h>
#include <linux/netlink.h>
#include <linux/sock_diag.h>
#include <linux/netlink_diag.h>
#include <linux/rhashtable.h>

#include "af_netlink.h"

static int sk_diag_dump_groups(struct sock *sk, struct sk_buff *nlskb)
{
        struct netlink_sock *nlk = nlk_sk(sk);

        if (nlk->groups == NULL)
                return 0;

        return nla_put(nlskb, NETLINK_DIAG_GROUPS, NLGRPSZ(nlk->ngroups),
                       nlk->groups);
}

static int sk_diag_put_flags(struct sock *sk, struct sk_buff *skb)
{
        struct netlink_sock *nlk = nlk_sk(sk);
        u32 flags = 0;

        if (nlk->cb_running)
                flags |= NDIAG_FLAG_CB_RUNNING;
        if (nlk_test_bit(RECV_PKTINFO, sk))
                flags |= NDIAG_FLAG_PKTINFO;
        if (nlk_test_bit(BROADCAST_SEND_ERROR, sk))
                flags |= NDIAG_FLAG_BROADCAST_ERROR;
        if (nlk_test_bit(RECV_NO_ENOBUFS, sk))
                flags |= NDIAG_FLAG_NO_ENOBUFS;
        if (nlk_test_bit(LISTEN_ALL_NSID, sk))
                flags |= NDIAG_FLAG_LISTEN_ALL_NSID;
        if (nlk_test_bit(CAP_ACK, sk))
                flags |= NDIAG_FLAG_CAP_ACK;

        return nla_put_u32(skb, NETLINK_DIAG_FLAGS, flags);
}

static int sk_diag_fill(struct sock *sk, struct sk_buff *skb,
                        struct netlink_diag_req *req,
                        u32 portid, u32 seq, u32 flags, u64 sk_ino)
{
        struct nlmsghdr *nlh;
        struct netlink_diag_msg *rep;
        struct netlink_sock *nlk = nlk_sk(sk);

        nlh = nlmsg_put(skb, portid, seq, SOCK_DIAG_BY_FAMILY, sizeof(*rep),
                        flags);
        if (!nlh)
                return -EMSGSIZE;

        rep = nlmsg_data(nlh);
        rep->ndiag_family        = AF_NETLINK;
        rep->ndiag_type                = sk->sk_type;
        rep->ndiag_protocol        = sk->sk_protocol;
        rep->ndiag_state        = sk->sk_state;

        rep->ndiag_ino                = sk_ino;
        rep->ndiag_portid        = nlk->portid;
        rep->ndiag_dst_portid        = nlk->dst_portid;
        rep->ndiag_dst_group        = nlk->dst_group;
        sock_diag_save_cookie(sk, rep->ndiag_cookie);

        if ((req->ndiag_show & NDIAG_SHOW_GROUPS) &&
            sk_diag_dump_groups(sk, skb))
                goto out_nlmsg_trim;

        if ((req->ndiag_show & NDIAG_SHOW_MEMINFO) &&
            sock_diag_put_meminfo(sk, skb, NETLINK_DIAG_MEMINFO))
                goto out_nlmsg_trim;

        if ((req->ndiag_show & NDIAG_SHOW_FLAGS) &&
            sk_diag_put_flags(sk, skb))
                goto out_nlmsg_trim;

        nlmsg_end(skb, nlh);
        return 0;

out_nlmsg_trim:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int __netlink_diag_dump(struct sk_buff *skb, struct netlink_callback *cb,
                                int protocol, int s_num)
{
        struct rhashtable_iter *hti = (void *)cb->args[2];
        struct netlink_table *tbl = &nl_table[protocol];
        struct net *net = sock_net(skb->sk);
        struct netlink_diag_req *req;
        struct netlink_sock *nlsk;
        unsigned long flags;
        struct sock *sk;
        int num = 2;
        int ret = 0;

        req = nlmsg_data(cb->nlh);

        if (s_num > 1)
                goto mc_list;

        num--;

        if (!hti) {
                hti = kmalloc_obj(*hti);
                if (!hti)
                        return -ENOMEM;

                cb->args[2] = (long)hti;
        }

        if (!s_num)
                rhashtable_walk_enter(&tbl->hash, hti);

        rhashtable_walk_start(hti);

        while ((nlsk = rhashtable_walk_next(hti))) {
                if (IS_ERR(nlsk)) {
                        ret = PTR_ERR(nlsk);
                        if (ret == -EAGAIN) {
                                ret = 0;
                                continue;
                        }
                        break;
                }

                sk = (struct sock *)nlsk;

                if (!net_eq(sock_net(sk), net))
                        continue;

                if (sk_diag_fill(sk, skb, req,
                                 NETLINK_CB(cb->skb).portid,
                                 cb->nlh->nlmsg_seq,
                                 NLM_F_MULTI,
                                 sock_i_ino(sk)) < 0) {
                        ret = 1;
                        break;
                }
        }

        rhashtable_walk_stop(hti);

        if (ret)
                goto done;

        rhashtable_walk_exit(hti);
        num++;

mc_list:
        read_lock_irqsave(&nl_table_lock, flags);
        sk_for_each_bound(sk, &tbl->mc_list) {
                if (sk_hashed(sk))
                        continue;
                if (!net_eq(sock_net(sk), net))
                        continue;
                if (num < s_num) {
                        num++;
                        continue;
                }

                if (sk_diag_fill(sk, skb, req,
                                 NETLINK_CB(cb->skb).portid,
                                 cb->nlh->nlmsg_seq,
                                 NLM_F_MULTI,
                                 sock_i_ino(sk)) < 0) {
                        ret = 1;
                        break;
                }
                num++;
        }
        read_unlock_irqrestore(&nl_table_lock, flags);

done:
        cb->args[0] = num;

        return ret;
}

static int netlink_diag_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct netlink_diag_req *req;
        int s_num = cb->args[0];
        int err = 0;

        req = nlmsg_data(cb->nlh);

        if (req->sdiag_protocol == NDIAG_PROTO_ALL) {
                int i;

                for (i = cb->args[1]; i < MAX_LINKS; i++) {
                        err = __netlink_diag_dump(skb, cb, i, s_num);
                        if (err)
                                break;
                        s_num = 0;
                }
                cb->args[1] = i;
        } else {
                if (req->sdiag_protocol >= MAX_LINKS)
                        return -ENOENT;

                err = __netlink_diag_dump(skb, cb, req->sdiag_protocol, s_num);
        }

        return err <= 0 ? err : skb->len;
}

static int netlink_diag_dump_done(struct netlink_callback *cb)
{
        struct rhashtable_iter *hti = (void *)cb->args[2];

        if (cb->args[0] == 1)
                rhashtable_walk_exit(hti);

        kfree(hti);

        return 0;
}

static int netlink_diag_handler_dump(struct sk_buff *skb, struct nlmsghdr *h)
{
        int hdrlen = sizeof(struct netlink_diag_req);
        struct net *net = sock_net(skb->sk);

        if (nlmsg_len(h) < hdrlen)
                return -EINVAL;

        if (h->nlmsg_flags & NLM_F_DUMP) {
                struct netlink_dump_control c = {
                        .dump = netlink_diag_dump,
                        .done = netlink_diag_dump_done,
                };
                return netlink_dump_start(net->diag_nlsk, skb, h, &c);
        } else
                return -EOPNOTSUPP;
}

static const struct sock_diag_handler netlink_diag_handler = {
        .owner = THIS_MODULE,
        .family = AF_NETLINK,
        .dump = netlink_diag_handler_dump,
};

static int __init netlink_diag_init(void)
{
        return sock_diag_register(&netlink_diag_handler);
}

static void __exit netlink_diag_exit(void)
{
        sock_diag_unregister(&netlink_diag_handler);
}

module_init(netlink_diag_init);
module_exit(netlink_diag_exit);
MODULE_DESCRIPTION("Netlink-based socket monitoring/diagnostic interface (sock_diag)");
MODULE_LICENSE("GPL");
MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 16 /* AF_NETLINK */);
















































































































































































































































   22 































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2014 Felix Fietkau <nbd@nbd.name>
 * Copyright (C) 2004 - 2009 Ivo van Doorn <IvDoorn@gmail.com>
 */

#ifndef _LINUX_BITFIELD_H
#define _LINUX_BITFIELD_H

#include <linux/build_bug.h>
#include <linux/compiler.h>
#include <linux/typecheck.h>
#include <asm/byteorder.h>

/*
 * Bitfield access macros
 *
 * FIELD_{GET,PREP} macros take as first parameter shifted mask
 * from which they extract the base mask and shift amount.
 * Mask must be a compilation time constant.
 * field_{get,prep} are variants that take a non-const mask.
 *
 * Example:
 *
 *  #include <linux/bitfield.h>
 *  #include <linux/bits.h>
 *
 *  #define REG_FIELD_A  GENMASK(6, 0)
 *  #define REG_FIELD_B  BIT(7)
 *  #define REG_FIELD_C  GENMASK(15, 8)
 *  #define REG_FIELD_D  GENMASK(31, 16)
 *
 * Get:
 *  a = FIELD_GET(REG_FIELD_A, reg);
 *  b = FIELD_GET(REG_FIELD_B, reg);
 *
 * Set:
 *  reg = FIELD_PREP(REG_FIELD_A, 1) |
 *          FIELD_PREP(REG_FIELD_B, 0) |
 *          FIELD_PREP(REG_FIELD_C, c) |
 *          FIELD_PREP(REG_FIELD_D, 0x40);
 *
 * Modify:
 *  FIELD_MODIFY(REG_FIELD_C, &reg, c);
 */

#define __bf_shf(x) (__builtin_ffsll(x) - 1)

#define __scalar_type_to_unsigned_cases(type)                                \
                unsigned type:        (unsigned type)0,                        \
                signed type:        (unsigned type)0

#define __unsigned_scalar_typeof(x) typeof(                                \
                _Generic((x),                                                \
                        char:        (unsigned char)0,                        \
                        __scalar_type_to_unsigned_cases(char),                \
                        __scalar_type_to_unsigned_cases(short),                \
                        __scalar_type_to_unsigned_cases(int),                \
                        __scalar_type_to_unsigned_cases(long),                \
                        __scalar_type_to_unsigned_cases(long long),        \
                        default: (x)))

#define __bf_cast_unsigned(type, x)        ((__unsigned_scalar_typeof(type))(x))

#define __BF_FIELD_CHECK_MASK(_mask, _val, _pfx)                        \
        ({                                                                \
                BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask),                \
                                 _pfx "mask is not constant");                \
                BUILD_BUG_ON_MSG((_mask) == 0, _pfx "mask is zero");        \
                BUILD_BUG_ON_MSG(__builtin_constant_p(_val) ?                \
                                 ~((_mask) >> __bf_shf(_mask)) &        \
                                        (0 + (_val)) : 0,                \
                                 _pfx "value too large for the field"); \
                __BUILD_BUG_ON_NOT_POWER_OF_2((_mask) +                        \
                                              (1ULL << __bf_shf(_mask))); \
        })

#define __BF_FIELD_CHECK_REG(mask, reg, pfx)                                \
        BUILD_BUG_ON_MSG(__bf_cast_unsigned(mask, mask) >                \
                         __bf_cast_unsigned(reg, ~0ull),                \
                         pfx "type of reg too small for mask")

#define __BF_FIELD_CHECK(mask, reg, val, pfx)                                \
        ({                                                                \
                __BF_FIELD_CHECK_MASK(mask, val, pfx);                        \
                __BF_FIELD_CHECK_REG(mask, reg, pfx);                        \
        })

#define __FIELD_PREP(mask, val, pfx)                                        \
        ({                                                                \
                __BF_FIELD_CHECK_MASK(mask, val, pfx);                        \
                ((typeof(mask))(val) << __bf_shf(mask)) & (mask);        \
        })

#define __FIELD_GET(mask, reg, pfx)                                        \
        ({                                                                \
                __BF_FIELD_CHECK_MASK(mask, 0U, pfx);                        \
                (typeof(mask))(((reg) & (mask)) >> __bf_shf(mask));        \
        })

/**
 * FIELD_MAX() - produce the maximum value representable by a field
 * @_mask: shifted mask defining the field's length and position
 *
 * FIELD_MAX() returns the maximum value that can be held in the field
 * specified by @_mask.
 */
#define FIELD_MAX(_mask)                                                \
        ({                                                                \
                __BF_FIELD_CHECK(_mask, 0ULL, 0ULL, "FIELD_MAX: ");        \
                (typeof(_mask))((_mask) >> __bf_shf(_mask));                \
        })

/**
 * FIELD_FIT() - check if value fits in the field
 * @_mask: shifted mask defining the field's length and position
 * @_val:  value to test against the field
 *
 * Return: true if @_val can fit inside @_mask, false if @_val is too big.
 */
#define FIELD_FIT(_mask, _val)                                                \
        ({                                                                \
                __BF_FIELD_CHECK(_mask, 0ULL, 0ULL, "FIELD_FIT: ");        \
                !((((typeof(_mask))_val) << __bf_shf(_mask)) & ~(_mask)); \
        })

/**
 * FIELD_PREP() - prepare a bitfield element
 * @_mask: shifted mask defining the field's length and position
 * @_val:  value to put in the field
 *
 * FIELD_PREP() masks and shifts up the value.  The result should
 * be combined with other fields of the bitfield using logical OR.
 */
#define FIELD_PREP(_mask, _val)                                                \
        ({                                                                \
                __BF_FIELD_CHECK_REG(_mask, 0ULL, "FIELD_PREP: ");        \
                __FIELD_PREP(_mask, _val, "FIELD_PREP: ");                \
        })

#define __BF_CHECK_POW2(n)        BUILD_BUG_ON_ZERO(((n) & ((n) - 1)) != 0)

/**
 * FIELD_PREP_CONST() - prepare a constant bitfield element
 * @_mask: shifted mask defining the field's length and position
 * @_val:  value to put in the field
 *
 * FIELD_PREP_CONST() masks and shifts up the value.  The result should
 * be combined with other fields of the bitfield using logical OR.
 *
 * Unlike FIELD_PREP() this is a constant expression and can therefore
 * be used in initializers. Error checking is less comfortable for this
 * version, and non-constant masks cannot be used.
 */
#define FIELD_PREP_CONST(_mask, _val)                                        \
        (                                                                \
                /* mask must be non-zero */                                \
                BUILD_BUG_ON_ZERO((_mask) == 0) +                        \
                /* check if value fits */                                \
                BUILD_BUG_ON_ZERO(~((_mask) >> __bf_shf(_mask)) & (_val)) + \
                /* check if mask is contiguous */                        \
                __BF_CHECK_POW2((_mask) + (1ULL << __bf_shf(_mask))) +        \
                /* and create the value */                                \
                (((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask))        \
        )

/**
 * FIELD_GET() - extract a bitfield element
 * @_mask: shifted mask defining the field's length and position
 * @_reg:  value of entire bitfield
 *
 * FIELD_GET() extracts the field specified by @_mask from the
 * bitfield passed in as @_reg by masking and shifting it down.
 */
#define FIELD_GET(_mask, _reg)                                                \
        ({                                                                \
                __BF_FIELD_CHECK_REG(_mask, _reg, "FIELD_GET: ");        \
                __FIELD_GET(_mask, _reg, "FIELD_GET: ");                \
        })

/**
 * FIELD_MODIFY() - modify a bitfield element
 * @_mask: shifted mask defining the field's length and position
 * @_reg_p: pointer to the memory that should be updated
 * @_val: value to store in the bitfield
 *
 * FIELD_MODIFY() modifies the set of bits in @_reg_p specified by @_mask,
 * by replacing them with the bitfield value passed in as @_val.
 */
#define FIELD_MODIFY(_mask, _reg_p, _val)                                                \
        ({                                                                                \
                typecheck_pointer(_reg_p);                                                \
                __BF_FIELD_CHECK(_mask, *(_reg_p), _val, "FIELD_MODIFY: ");                \
                *(_reg_p) &= ~(_mask);                                                        \
                *(_reg_p) |= (((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask));        \
        })

extern void __compiletime_error("value doesn't fit into mask")
__field_overflow(void);
extern void __compiletime_error("bad bitfield mask")
__bad_mask(void);
static __always_inline u64 field_multiplier(u64 field)
{
        if ((field | (field - 1)) & ((field | (field - 1)) + 1))
                __bad_mask();
        return field & -field;
}
static __always_inline u64 field_mask(u64 field)
{
        return field / field_multiplier(field);
}
#define field_max(field)        ((typeof(field))field_mask(field))
#define ____MAKE_OP(type,base,to,from)                                        \
static __always_inline __##type __must_check type##_encode_bits(base v, base field)        \
{                                                                        \
        if (__builtin_constant_p(v) && (v & ~field_mask(field)))        \
                __field_overflow();                                        \
        return to((v & field_mask(field)) * field_multiplier(field));        \
}                                                                        \
static __always_inline __##type __must_check type##_replace_bits(__##type old,        \
                                                        base val, base field)        \
{                                                                        \
        return (old & ~to(field)) | type##_encode_bits(val, field);        \
}                                                                        \
static __always_inline void type##p_replace_bits(__##type *p,                \
                                        base val, base field)                \
{                                                                        \
        *p = (*p & ~to(field)) | type##_encode_bits(val, field);        \
}                                                                        \
static __always_inline base __must_check type##_get_bits(__##type v, base field)        \
{                                                                        \
        return (from(v) & field)/field_multiplier(field);                \
}
#define __MAKE_OP(size)                                                        \
        ____MAKE_OP(le##size,u##size,cpu_to_le##size,le##size##_to_cpu)        \
        ____MAKE_OP(be##size,u##size,cpu_to_be##size,be##size##_to_cpu)        \
        ____MAKE_OP(u##size,u##size,,)
____MAKE_OP(u8,u8,,)
__MAKE_OP(16)
__MAKE_OP(32)
__MAKE_OP(64)
#undef __MAKE_OP
#undef ____MAKE_OP

#define __field_prep(mask, val)                                                \
        ({                                                                \
                auto __mask = (mask);                                        \
                typeof(__mask) __val = (val);                                \
                unsigned int __shift = BITS_PER_TYPE(__mask) <= 32 ?        \
                                       __ffs(__mask) : __ffs64(__mask);        \
                (__val << __shift) & __mask;                                \
        })

#define __field_get(mask, reg)                                                \
        ({                                                                \
                auto __mask = (mask);                                        \
                typeof(__mask) __reg =  (reg);                                \
                unsigned int __shift = BITS_PER_TYPE(__mask) <= 32 ?        \
                                       __ffs(__mask) : __ffs64(__mask);        \
                (__reg & __mask) >> __shift;                                \
        })

/**
 * field_prep() - prepare a bitfield element
 * @mask: shifted mask defining the field's length and position, must be
 *        non-zero
 * @val:  value to put in the field
 *
 * Return: field value masked and shifted to its final destination
 *
 * field_prep() masks and shifts up the value.  The result should be
 * combined with other fields of the bitfield using logical OR.
 * Unlike FIELD_PREP(), @mask is not limited to a compile-time constant.
 * Typical usage patterns are a value stored in a table, or calculated by
 * shifting a constant by a variable number of bits.
 * If you want to ensure that @mask is a compile-time constant, please use
 * FIELD_PREP() directly instead.
 */
#define field_prep(mask, val)                                                \
        (__builtin_constant_p(mask) ? __FIELD_PREP(mask, val, "field_prep: ") \
                                    : __field_prep(mask, val))

/**
 * field_get() - extract a bitfield element
 * @mask: shifted mask defining the field's length and position, must be
 *        non-zero
 * @reg:  value of entire bitfield
 *
 * Return: extracted field value
 *
 * field_get() extracts the field specified by @mask from the
 * bitfield passed in as @reg by masking and shifting it down.
 * Unlike FIELD_GET(), @mask is not limited to a compile-time constant.
 * Typical usage patterns are a value stored in a table, or calculated by
 * shifting a constant by a variable number of bits.
 * If you want to ensure that @mask is a compile-time constant, please use
 * FIELD_GET() directly instead.
 */
#define field_get(mask, reg)                                                \
        (__builtin_constant_p(mask) ? __FIELD_GET(mask, reg, "field_get: ") \
                                    : __field_get(mask, reg))

#endif



























































































































    8 


    8 


    5 
    5 










    8 


    8 


    7 
    7 











    2 






    2 


    2 















    7 


    7 


    7 
    7 

















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C)2003,2004 USAGI/WIDE Project
 *
 * Authors        Mitsuru KANDA  <mk@linux-ipv6.org>
 *                YOSHIFUJI Hideaki <yoshfuji@linux-ipv6.org>
 */

#define pr_fmt(fmt) "IPv6: " fmt

#include <linux/icmpv6.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <net/ipv6.h>
#include <net/protocol.h>
#include <net/xfrm.h>

static struct xfrm6_tunnel __rcu *tunnel6_handlers __read_mostly;
static struct xfrm6_tunnel __rcu *tunnel46_handlers __read_mostly;
static struct xfrm6_tunnel __rcu *tunnelmpls6_handlers __read_mostly;
static DEFINE_MUTEX(tunnel6_mutex);

static inline int xfrm6_tunnel_mpls_supported(void)
{
        return IS_ENABLED(CONFIG_MPLS);
}

int xfrm6_tunnel_register(struct xfrm6_tunnel *handler, unsigned short family)
{
        struct xfrm6_tunnel __rcu **pprev;
        struct xfrm6_tunnel *t;
        int ret = -EEXIST;
        int priority = handler->priority;

        mutex_lock(&tunnel6_mutex);

        switch (family) {
        case AF_INET6:
                pprev = &tunnel6_handlers;
                break;
        case AF_INET:
                pprev = &tunnel46_handlers;
                break;
        case AF_MPLS:
                pprev = &tunnelmpls6_handlers;
                break;
        default:
                goto err;
        }

        for (; (t = rcu_dereference_protected(*pprev,
                        lockdep_is_held(&tunnel6_mutex))) != NULL;
             pprev = &t->next) {
                if (t->priority > priority)
                        break;
                if (t->priority == priority)
                        goto err;
        }

        handler->next = *pprev;
        rcu_assign_pointer(*pprev, handler);

        ret = 0;

err:
        mutex_unlock(&tunnel6_mutex);

        return ret;
}
EXPORT_SYMBOL(xfrm6_tunnel_register);

int xfrm6_tunnel_deregister(struct xfrm6_tunnel *handler, unsigned short family)
{
        struct xfrm6_tunnel __rcu **pprev;
        struct xfrm6_tunnel *t;
        int ret = -ENOENT;

        mutex_lock(&tunnel6_mutex);

        switch (family) {
        case AF_INET6:
                pprev = &tunnel6_handlers;
                break;
        case AF_INET:
                pprev = &tunnel46_handlers;
                break;
        case AF_MPLS:
                pprev = &tunnelmpls6_handlers;
                break;
        default:
                goto err;
        }

        for (; (t = rcu_dereference_protected(*pprev,
                        lockdep_is_held(&tunnel6_mutex))) != NULL;
             pprev = &t->next) {
                if (t == handler) {
                        *pprev = handler->next;
                        ret = 0;
                        break;
                }
        }

err:
        mutex_unlock(&tunnel6_mutex);

        synchronize_net();

        return ret;
}
EXPORT_SYMBOL(xfrm6_tunnel_deregister);

#define for_each_tunnel_rcu(head, handler)                \
        for (handler = rcu_dereference(head);                \
             handler != NULL;                                \
             handler = rcu_dereference(handler->next))        \

static int tunnelmpls6_rcv(struct sk_buff *skb)
{
        struct xfrm6_tunnel *handler;

        if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
                goto drop;

        for_each_tunnel_rcu(tunnelmpls6_handlers, handler)
                if (!handler->handler(skb))
                        return 0;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

drop:
        kfree_skb(skb);
        return 0;
}

static int tunnel6_rcv(struct sk_buff *skb)
{
        struct xfrm6_tunnel *handler;

        if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
                goto drop;

        for_each_tunnel_rcu(tunnel6_handlers, handler)
                if (!handler->handler(skb))
                        return 0;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

drop:
        kfree_skb(skb);
        return 0;
}

#if IS_ENABLED(CONFIG_INET6_XFRM_TUNNEL)
static int tunnel6_rcv_cb(struct sk_buff *skb, u8 proto, int err)
{
        struct xfrm6_tunnel __rcu *head;
        struct xfrm6_tunnel *handler;
        int ret;

        head = (proto == IPPROTO_IPV6) ? tunnel6_handlers : tunnel46_handlers;

        for_each_tunnel_rcu(head, handler) {
                if (handler->cb_handler) {
                        ret = handler->cb_handler(skb, err);
                        if (ret <= 0)
                                return ret;
                }
        }

        return 0;
}

static const struct xfrm_input_afinfo tunnel6_input_afinfo = {
        .family                =        AF_INET6,
        .is_ipip        =        true,
        .callback        =        tunnel6_rcv_cb,
};
#endif

static int tunnel46_rcv(struct sk_buff *skb)
{
        struct xfrm6_tunnel *handler;

        if (!pskb_may_pull(skb, sizeof(struct iphdr)))
                goto drop;

        for_each_tunnel_rcu(tunnel46_handlers, handler)
                if (!handler->handler(skb))
                        return 0;

        icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);

drop:
        kfree_skb(skb);
        return 0;
}

static int tunnel6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                        u8 type, u8 code, int offset, __be32 info)
{
        struct xfrm6_tunnel *handler;

        for_each_tunnel_rcu(tunnel6_handlers, handler)
                if (!handler->err_handler(skb, opt, type, code, offset, info))
                        return 0;

        return -ENOENT;
}

static int tunnel46_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                         u8 type, u8 code, int offset, __be32 info)
{
        struct xfrm6_tunnel *handler;

        for_each_tunnel_rcu(tunnel46_handlers, handler)
                if (!handler->err_handler(skb, opt, type, code, offset, info))
                        return 0;

        return -ENOENT;
}

static int tunnelmpls6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
                           u8 type, u8 code, int offset, __be32 info)
{
        struct xfrm6_tunnel *handler;

        for_each_tunnel_rcu(tunnelmpls6_handlers, handler)
                if (!handler->err_handler(skb, opt, type, code, offset, info))
                        return 0;

        return -ENOENT;
}

static const struct inet6_protocol tunnel6_protocol = {
        .handler        = tunnel6_rcv,
        .err_handler        = tunnel6_err,
        .flags          = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
};

static const struct inet6_protocol tunnel46_protocol = {
        .handler        = tunnel46_rcv,
        .err_handler        = tunnel46_err,
        .flags          = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
};

static const struct inet6_protocol tunnelmpls6_protocol = {
        .handler        = tunnelmpls6_rcv,
        .err_handler        = tunnelmpls6_err,
        .flags          = INET6_PROTO_NOPOLICY|INET6_PROTO_FINAL,
};

static int __init tunnel6_init(void)
{
        if (inet6_add_protocol(&tunnel6_protocol, IPPROTO_IPV6)) {
                pr_err("%s: can't add protocol\n", __func__);
                return -EAGAIN;
        }
        if (inet6_add_protocol(&tunnel46_protocol, IPPROTO_IPIP)) {
                pr_err("%s: can't add protocol\n", __func__);
                inet6_del_protocol(&tunnel6_protocol, IPPROTO_IPV6);
                return -EAGAIN;
        }
        if (xfrm6_tunnel_mpls_supported() &&
            inet6_add_protocol(&tunnelmpls6_protocol, IPPROTO_MPLS)) {
                pr_err("%s: can't add protocol\n", __func__);
                inet6_del_protocol(&tunnel6_protocol, IPPROTO_IPV6);
                inet6_del_protocol(&tunnel46_protocol, IPPROTO_IPIP);
                return -EAGAIN;
        }
#if IS_ENABLED(CONFIG_INET6_XFRM_TUNNEL)
        if (xfrm_input_register_afinfo(&tunnel6_input_afinfo)) {
                pr_err("%s: can't add input afinfo\n", __func__);
                inet6_del_protocol(&tunnel6_protocol, IPPROTO_IPV6);
                inet6_del_protocol(&tunnel46_protocol, IPPROTO_IPIP);
                if (xfrm6_tunnel_mpls_supported())
                        inet6_del_protocol(&tunnelmpls6_protocol, IPPROTO_MPLS);
                return -EAGAIN;
        }
#endif
        return 0;
}

static void __exit tunnel6_fini(void)
{
#if IS_ENABLED(CONFIG_INET6_XFRM_TUNNEL)
        if (xfrm_input_unregister_afinfo(&tunnel6_input_afinfo))
                pr_err("%s: can't remove input afinfo\n", __func__);
#endif
        if (inet6_del_protocol(&tunnel46_protocol, IPPROTO_IPIP))
                pr_err("%s: can't remove protocol\n", __func__);
        if (inet6_del_protocol(&tunnel6_protocol, IPPROTO_IPV6))
                pr_err("%s: can't remove protocol\n", __func__);
        if (xfrm6_tunnel_mpls_supported() &&
            inet6_del_protocol(&tunnelmpls6_protocol, IPPROTO_MPLS))
                pr_err("%s: can't remove protocol\n", __func__);
}

module_init(tunnel6_init);
module_exit(tunnel6_fini);
MODULE_DESCRIPTION("IP-in-IPv6 tunnel driver");
MODULE_LICENSE("GPL");













































































































































































































































































































































































































  131 


























  108 































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 







    2 







    2 







    2 



















































































































































   14 



























   14 







   14 



   14 










   14 











































   14 









   14 













































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
/*
 * net/tipc/link.c: TIPC link code
 *
 * Copyright (c) 1996-2007, 2012-2016, Ericsson AB
 * Copyright (c) 2004-2007, 2010-2013, Wind River Systems
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "core.h"
#include "subscr.h"
#include "link.h"
#include "bcast.h"
#include "socket.h"
#include "name_distr.h"
#include "discover.h"
#include "netlink.h"
#include "monitor.h"
#include "trace.h"
#include "crypto.h"

#include <linux/pkt_sched.h>

struct tipc_stats {
        u32 sent_pkts;
        u32 recv_pkts;
        u32 sent_states;
        u32 recv_states;
        u32 sent_probes;
        u32 recv_probes;
        u32 sent_nacks;
        u32 recv_nacks;
        u32 sent_acks;
        u32 sent_bundled;
        u32 sent_bundles;
        u32 recv_bundled;
        u32 recv_bundles;
        u32 retransmitted;
        u32 sent_fragmented;
        u32 sent_fragments;
        u32 recv_fragmented;
        u32 recv_fragments;
        u32 link_congs;                /* # port sends blocked by congestion */
        u32 deferred_recv;
        u32 duplicates;
        u32 max_queue_sz;        /* send queue size high water mark */
        u32 accu_queue_sz;        /* used for send queue size profiling */
        u32 queue_sz_counts;        /* used for send queue size profiling */
        u32 msg_length_counts;        /* used for message length profiling */
        u32 msg_lengths_total;        /* used for message length profiling */
        u32 msg_length_profile[7]; /* used for msg. length profiling */
};

/**
 * struct tipc_link - TIPC link data structure
 * @addr: network address of link's peer node
 * @name: link name character string
 * @net: pointer to namespace struct
 * @peer_session: link session # being used by peer end of link
 * @peer_bearer_id: bearer id used by link's peer endpoint
 * @bearer_id: local bearer id used by link
 * @tolerance: minimum link continuity loss needed to reset link [in ms]
 * @abort_limit: # of unacknowledged continuity probes needed to reset link
 * @state: current state of link FSM
 * @peer_caps: bitmap describing capabilities of peer node
 * @silent_intv_cnt: # of timer intervals without any reception from peer
 * @priority: current link priority
 * @net_plane: current link network plane ('A' through 'H')
 * @mon_state: cookie with information needed by link monitor
 * @mtu: current maximum packet size for this link
 * @advertised_mtu: advertised own mtu when link is being established
 * @backlogq: queue for messages waiting to be sent
 * @ackers: # of peers that needs to ack each packet before it can be released
 * @acked: # last packet acked by a certain peer. Used for broadcast.
 * @rcv_nxt: next sequence number to expect for inbound messages
 * @inputq: buffer queue for messages to be delivered upwards
 * @namedq: buffer queue for name table messages to be delivered upwards
 * @wakeupq: linked list of wakeup msgs waiting for link congestion to abate
 * @reasm_buf: head of partially reassembled inbound message fragments
 * @stats: collects statistics regarding link activity
 * @session: session to be used by link
 * @snd_nxt_state: next send seq number
 * @rcv_nxt_state: next rcv seq number
 * @in_session: have received ACTIVATE_MSG from peer
 * @active: link is active
 * @if_name: associated interface name
 * @rst_cnt: link reset counter
 * @drop_point: seq number for failover handling (FIXME)
 * @failover_reasm_skb: saved failover msg ptr (FIXME)
 * @failover_deferdq: deferred message queue for failover processing (FIXME)
 * @transmq: the link's transmit queue
 * @backlog: link's backlog by priority (importance)
 * @snd_nxt: next sequence number to be used
 * @rcv_unacked: # messages read by user, but not yet acked back to peer
 * @deferdq: deferred receive queue
 * @window: sliding window size for congestion handling
 * @min_win: minimal send window to be used by link
 * @ssthresh: slow start threshold for congestion handling
 * @max_win: maximal send window to be used by link
 * @cong_acks: congestion acks for congestion avoidance (FIXME)
 * @checkpoint: seq number for congestion window size handling
 * @reasm_tnlmsg: fragmentation/reassembly area for tunnel protocol message
 * @last_gap: last gap ack blocks for bcast (FIXME)
 * @last_ga: ptr to gap ack blocks
 * @bc_rcvlink: the peer specific link used for broadcast reception
 * @bc_sndlink: the namespace global link used for broadcast sending
 * @nack_state: bcast nack state
 * @bc_peer_is_up: peer has acked the bcast init msg
 */
struct tipc_link {
        u32 addr;
        char name[TIPC_MAX_LINK_NAME];
        struct net *net;

        /* Management and link supervision data */
        u16 peer_session;
        u16 session;
        u16 snd_nxt_state;
        u16 rcv_nxt_state;
        u32 peer_bearer_id;
        u32 bearer_id;
        u32 tolerance;
        u32 abort_limit;
        u32 state;
        u16 peer_caps;
        bool in_session;
        bool active;
        u32 silent_intv_cnt;
        char if_name[TIPC_MAX_IF_NAME];
        u32 priority;
        char net_plane;
        struct tipc_mon_state mon_state;
        u16 rst_cnt;

        /* Failover/synch */
        u16 drop_point;
        struct sk_buff *failover_reasm_skb;
        struct sk_buff_head failover_deferdq;

        /* Max packet negotiation */
        u16 mtu;
        u16 advertised_mtu;

        /* Sending */
        struct sk_buff_head transmq;
        struct sk_buff_head backlogq;
        struct {
                u16 len;
                u16 limit;
                struct sk_buff *target_bskb;
        } backlog[5];
        u16 snd_nxt;

        /* Reception */
        u16 rcv_nxt;
        u32 rcv_unacked;
        struct sk_buff_head deferdq;
        struct sk_buff_head *inputq;
        struct sk_buff_head *namedq;

        /* Congestion handling */
        struct sk_buff_head wakeupq;
        u16 window;
        u16 min_win;
        u16 ssthresh;
        u16 max_win;
        u16 cong_acks;
        u16 checkpoint;

        /* Fragmentation/reassembly */
        struct sk_buff *reasm_buf;
        struct sk_buff *reasm_tnlmsg;

        /* Broadcast */
        u16 ackers;
        u16 acked;
        u16 last_gap;
        struct tipc_gap_ack_blks *last_ga;
        struct tipc_link *bc_rcvlink;
        struct tipc_link *bc_sndlink;
        u8 nack_state;
        bool bc_peer_is_up;

        /* Statistics */
        struct tipc_stats stats;
};

/*
 * Error message prefixes
 */
static const char *link_co_err = "Link tunneling error, ";
static const char *link_rst_msg = "Resetting link ";

/* Send states for broadcast NACKs
 */
enum {
        BC_NACK_SND_CONDITIONAL,
        BC_NACK_SND_UNCONDITIONAL,
        BC_NACK_SND_SUPPRESS,
};

#define TIPC_BC_RETR_LIM  (jiffies + msecs_to_jiffies(10))
#define TIPC_UC_RETR_TIME (jiffies + msecs_to_jiffies(1))

/* Link FSM states:
 */
enum {
        LINK_ESTABLISHED     = 0xe,
        LINK_ESTABLISHING    = 0xe  << 4,
        LINK_RESET           = 0x1  << 8,
        LINK_RESETTING       = 0x2  << 12,
        LINK_PEER_RESET      = 0xd  << 16,
        LINK_FAILINGOVER     = 0xf  << 20,
        LINK_SYNCHING        = 0xc  << 24
};

static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
                               struct sk_buff_head *xmitq);
static void tipc_link_build_proto_msg(struct tipc_link *l, int mtyp, bool probe,
                                      bool probe_reply, u16 rcvgap,
                                      int tolerance, int priority,
                                      struct sk_buff_head *xmitq);
static void link_print(struct tipc_link *l, const char *str);
static int tipc_link_build_nack_msg(struct tipc_link *l,
                                    struct sk_buff_head *xmitq);
static void tipc_link_build_bc_init_msg(struct tipc_link *l,
                                        struct sk_buff_head *xmitq);
static u8 __tipc_build_gap_ack_blks(struct tipc_gap_ack_blks *ga,
                                    struct tipc_link *l, u8 start_index);
static u16 tipc_build_gap_ack_blks(struct tipc_link *l, struct tipc_msg *hdr);
static int tipc_link_advance_transmq(struct tipc_link *l, struct tipc_link *r,
                                     u16 acked, u16 gap,
                                     struct tipc_gap_ack_blks *ga,
                                     struct sk_buff_head *xmitq,
                                     bool *retransmitted, int *rc);
static void tipc_link_update_cwin(struct tipc_link *l, int released,
                                  bool retransmitted);
/*
 *  Simple non-static link routines (i.e. referenced outside this file)
 */
bool tipc_link_is_up(struct tipc_link *l)
{
        return l->state & (LINK_ESTABLISHED | LINK_SYNCHING);
}

bool tipc_link_peer_is_down(struct tipc_link *l)
{
        return l->state == LINK_PEER_RESET;
}

bool tipc_link_is_reset(struct tipc_link *l)
{
        return l->state & (LINK_RESET | LINK_FAILINGOVER | LINK_ESTABLISHING);
}

bool tipc_link_is_establishing(struct tipc_link *l)
{
        return l->state == LINK_ESTABLISHING;
}

bool tipc_link_is_synching(struct tipc_link *l)
{
        return l->state == LINK_SYNCHING;
}

bool tipc_link_is_failingover(struct tipc_link *l)
{
        return l->state == LINK_FAILINGOVER;
}

bool tipc_link_is_blocked(struct tipc_link *l)
{
        return l->state & (LINK_RESETTING | LINK_PEER_RESET | LINK_FAILINGOVER);
}

static bool link_is_bc_sndlink(struct tipc_link *l)
{
        return !l->bc_sndlink;
}

static bool link_is_bc_rcvlink(struct tipc_link *l)
{
        return ((l->bc_rcvlink == l) && !link_is_bc_sndlink(l));
}

void tipc_link_set_active(struct tipc_link *l, bool active)
{
        l->active = active;
}

u32 tipc_link_id(struct tipc_link *l)
{
        return l->peer_bearer_id << 16 | l->bearer_id;
}

int tipc_link_min_win(struct tipc_link *l)
{
        return l->min_win;
}

int tipc_link_max_win(struct tipc_link *l)
{
        return l->max_win;
}

int tipc_link_prio(struct tipc_link *l)
{
        return l->priority;
}

unsigned long tipc_link_tolerance(struct tipc_link *l)
{
        return l->tolerance;
}

struct sk_buff_head *tipc_link_inputq(struct tipc_link *l)
{
        return l->inputq;
}

char tipc_link_plane(struct tipc_link *l)
{
        return l->net_plane;
}

struct net *tipc_link_net(struct tipc_link *l)
{
        return l->net;
}

void tipc_link_update_caps(struct tipc_link *l, u16 capabilities)
{
        l->peer_caps = capabilities;
}

void tipc_link_add_bc_peer(struct tipc_link *snd_l,
                           struct tipc_link *uc_l,
                           struct sk_buff_head *xmitq)
{
        struct tipc_link *rcv_l = uc_l->bc_rcvlink;

        snd_l->ackers++;
        rcv_l->acked = snd_l->snd_nxt - 1;
        snd_l->state = LINK_ESTABLISHED;
        tipc_link_build_bc_init_msg(uc_l, xmitq);
}

void tipc_link_remove_bc_peer(struct tipc_link *snd_l,
                              struct tipc_link *rcv_l,
                              struct sk_buff_head *xmitq)
{
        u16 ack = snd_l->snd_nxt - 1;

        snd_l->ackers--;
        rcv_l->bc_peer_is_up = true;
        rcv_l->state = LINK_ESTABLISHED;
        tipc_link_bc_ack_rcv(rcv_l, ack, 0, NULL, xmitq, NULL);
        trace_tipc_link_reset(rcv_l, TIPC_DUMP_ALL, "bclink removed!");
        tipc_link_reset(rcv_l);
        rcv_l->state = LINK_RESET;
        if (!snd_l->ackers) {
                trace_tipc_link_reset(snd_l, TIPC_DUMP_ALL, "zero ackers!");
                tipc_link_reset(snd_l);
                snd_l->state = LINK_RESET;
                __skb_queue_purge(xmitq);
        }
}

int tipc_link_bc_peers(struct tipc_link *l)
{
        return l->ackers;
}

static u16 link_bc_rcv_gap(struct tipc_link *l)
{
        struct sk_buff *skb = skb_peek(&l->deferdq);
        u16 gap = 0;

        if (more(l->snd_nxt, l->rcv_nxt))
                gap = l->snd_nxt - l->rcv_nxt;
        if (skb)
                gap = buf_seqno(skb) - l->rcv_nxt;
        return gap;
}

void tipc_link_set_mtu(struct tipc_link *l, int mtu)
{
        l->mtu = mtu;
}

int tipc_link_mtu(struct tipc_link *l)
{
        return l->mtu;
}

int tipc_link_mss(struct tipc_link *l)
{
#ifdef CONFIG_TIPC_CRYPTO
        return l->mtu - INT_H_SIZE - EMSG_OVERHEAD;
#else
        return l->mtu - INT_H_SIZE;
#endif
}

u16 tipc_link_rcv_nxt(struct tipc_link *l)
{
        return l->rcv_nxt;
}

u16 tipc_link_acked(struct tipc_link *l)
{
        return l->acked;
}

char *tipc_link_name(struct tipc_link *l)
{
        return l->name;
}

u32 tipc_link_state(struct tipc_link *l)
{
        return l->state;
}

/**
 * tipc_link_create - create a new link
 * @net: pointer to associated network namespace
 * @if_name: associated interface name
 * @bearer_id: id (index) of associated bearer
 * @tolerance: link tolerance to be used by link
 * @net_plane: network plane (A,B,c..) this link belongs to
 * @mtu: mtu to be advertised by link
 * @priority: priority to be used by link
 * @min_win: minimal send window to be used by link
 * @max_win: maximal send window to be used by link
 * @session: session to be used by link
 * @peer: node id of peer node
 * @peer_caps: bitmap describing peer node capabilities
 * @bc_sndlink: the namespace global link used for broadcast sending
 * @bc_rcvlink: the peer specific link used for broadcast reception
 * @inputq: queue to put messages ready for delivery
 * @namedq: queue to put binding table update messages ready for delivery
 * @link: return value, pointer to put the created link
 * @self: local unicast link id
 * @peer_id: 128-bit ID of peer
 *
 * Return: true if link was created, otherwise false
 */
bool tipc_link_create(struct net *net, char *if_name, int bearer_id,
                      int tolerance, char net_plane, u32 mtu, int priority,
                      u32 min_win, u32 max_win, u32 session, u32 self,
                      u32 peer, u8 *peer_id, u16 peer_caps,
                      struct tipc_link *bc_sndlink,
                      struct tipc_link *bc_rcvlink,
                      struct sk_buff_head *inputq,
                      struct sk_buff_head *namedq,
                      struct tipc_link **link)
{
        char peer_str[NODE_ID_STR_LEN] = {0,};
        char self_str[NODE_ID_STR_LEN] = {0,};
        struct tipc_link *l;

        l = kzalloc_obj(*l, GFP_ATOMIC);
        if (!l)
                return false;
        *link = l;
        l->session = session;

        /* Set link name for unicast links only */
        if (peer_id) {
                if (tipc_nodeid2string(self_str, tipc_own_id(net)) > NODE_ID_LEN)
                        sprintf(self_str, "%x", self);
                if (tipc_nodeid2string(peer_str, peer_id) > NODE_ID_LEN)
                        sprintf(peer_str, "%x", peer);
        }
        /* Peer i/f name will be completed by reset/activate message */
        snprintf(l->name, sizeof(l->name), "%s:%s-%s:unknown",
                 self_str, if_name, peer_str);

        strcpy(l->if_name, if_name);
        l->addr = peer;
        l->peer_caps = peer_caps;
        l->net = net;
        l->in_session = false;
        l->bearer_id = bearer_id;
        l->tolerance = tolerance;
        if (bc_rcvlink)
                bc_rcvlink->tolerance = tolerance;
        l->net_plane = net_plane;
        l->advertised_mtu = mtu;
        l->mtu = mtu;
        l->priority = priority;
        tipc_link_set_queue_limits(l, min_win, max_win);
        l->ackers = 1;
        l->bc_sndlink = bc_sndlink;
        l->bc_rcvlink = bc_rcvlink;
        l->inputq = inputq;
        l->namedq = namedq;
        l->state = LINK_RESETTING;
        __skb_queue_head_init(&l->transmq);
        __skb_queue_head_init(&l->backlogq);
        __skb_queue_head_init(&l->deferdq);
        __skb_queue_head_init(&l->failover_deferdq);
        skb_queue_head_init(&l->wakeupq);
        skb_queue_head_init(l->inputq);
        return true;
}

/**
 * tipc_link_bc_create - create new link to be used for broadcast
 * @net: pointer to associated network namespace
 * @mtu: mtu to be used initially if no peers
 * @min_win: minimal send window to be used by link
 * @max_win: maximal send window to be used by link
 * @inputq: queue to put messages ready for delivery
 * @namedq: queue to put binding table update messages ready for delivery
 * @link: return value, pointer to put the created link
 * @ownnode: identity of own node
 * @peer: node id of peer node
 * @peer_id: 128-bit ID of peer
 * @peer_caps: bitmap describing peer node capabilities
 * @bc_sndlink: the namespace global link used for broadcast sending
 *
 * Return: true if link was created, otherwise false
 */
bool tipc_link_bc_create(struct net *net, u32 ownnode, u32 peer, u8 *peer_id,
                         int mtu, u32 min_win, u32 max_win, u16 peer_caps,
                         struct sk_buff_head *inputq,
                         struct sk_buff_head *namedq,
                         struct tipc_link *bc_sndlink,
                         struct tipc_link **link)
{
        struct tipc_link *l;

        if (!tipc_link_create(net, "", MAX_BEARERS, 0, 'Z', mtu, 0, min_win,
                              max_win, 0, ownnode, peer, NULL, peer_caps,
                              bc_sndlink, NULL, inputq, namedq, link))
                return false;

        l = *link;
        if (peer_id) {
                char peer_str[NODE_ID_STR_LEN] = {0,};

                if (tipc_nodeid2string(peer_str, peer_id) > NODE_ID_LEN)
                        sprintf(peer_str, "%x", peer);
                /* Broadcast receiver link name: "broadcast-link:<peer>" */
                snprintf(l->name, sizeof(l->name), "%s:%s", tipc_bclink_name,
                         peer_str);
        } else {
                strcpy(l->name, tipc_bclink_name);
        }
        trace_tipc_link_reset(l, TIPC_DUMP_ALL, "bclink created!");
        tipc_link_reset(l);
        l->state = LINK_RESET;
        l->ackers = 0;
        l->bc_rcvlink = l;

        /* Broadcast send link is always up */
        if (link_is_bc_sndlink(l))
                l->state = LINK_ESTABLISHED;

        /* Disable replicast if even a single peer doesn't support it */
        if (link_is_bc_rcvlink(l) && !(peer_caps & TIPC_BCAST_RCAST))
                tipc_bcast_toggle_rcast(net, false);

        return true;
}

/**
 * tipc_link_fsm_evt - link finite state machine
 * @l: pointer to link
 * @evt: state machine event to be processed
 */
int tipc_link_fsm_evt(struct tipc_link *l, int evt)
{
        int rc = 0;
        int old_state = l->state;

        switch (l->state) {
        case LINK_RESETTING:
                switch (evt) {
                case LINK_PEER_RESET_EVT:
                        l->state = LINK_PEER_RESET;
                        break;
                case LINK_RESET_EVT:
                        l->state = LINK_RESET;
                        break;
                case LINK_FAILURE_EVT:
                case LINK_FAILOVER_BEGIN_EVT:
                case LINK_ESTABLISH_EVT:
                case LINK_FAILOVER_END_EVT:
                case LINK_SYNCH_BEGIN_EVT:
                case LINK_SYNCH_END_EVT:
                default:
                        goto illegal_evt;
                }
                break;
        case LINK_RESET:
                switch (evt) {
                case LINK_PEER_RESET_EVT:
                        l->state = LINK_ESTABLISHING;
                        break;
                case LINK_FAILOVER_BEGIN_EVT:
                        l->state = LINK_FAILINGOVER;
                        break;
                case LINK_FAILURE_EVT:
                case LINK_RESET_EVT:
                case LINK_ESTABLISH_EVT:
                case LINK_FAILOVER_END_EVT:
                        break;
                case LINK_SYNCH_BEGIN_EVT:
                case LINK_SYNCH_END_EVT:
                default:
                        goto illegal_evt;
                }
                break;
        case LINK_PEER_RESET:
                switch (evt) {
                case LINK_RESET_EVT:
                        l->state = LINK_ESTABLISHING;
                        break;
                case LINK_PEER_RESET_EVT:
                case LINK_ESTABLISH_EVT:
                case LINK_FAILURE_EVT:
                        break;
                case LINK_SYNCH_BEGIN_EVT:
                case LINK_SYNCH_END_EVT:
                case LINK_FAILOVER_BEGIN_EVT:
                case LINK_FAILOVER_END_EVT:
                default:
                        goto illegal_evt;
                }
                break;
        case LINK_FAILINGOVER:
                switch (evt) {
                case LINK_FAILOVER_END_EVT:
                        l->state = LINK_RESET;
                        break;
                case LINK_PEER_RESET_EVT:
                case LINK_RESET_EVT:
                case LINK_ESTABLISH_EVT:
                case LINK_FAILURE_EVT:
                        break;
                case LINK_FAILOVER_BEGIN_EVT:
                case LINK_SYNCH_BEGIN_EVT:
                case LINK_SYNCH_END_EVT:
                default:
                        goto illegal_evt;
                }
                break;
        case LINK_ESTABLISHING:
                switch (evt) {
                case LINK_ESTABLISH_EVT:
                        l->state = LINK_ESTABLISHED;
                        break;
                case LINK_FAILOVER_BEGIN_EVT:
                        l->state = LINK_FAILINGOVER;
                        break;
                case LINK_RESET_EVT:
                        l->state = LINK_RESET;
                        break;
                case LINK_FAILURE_EVT:
                case LINK_PEER_RESET_EVT:
                case LINK_SYNCH_BEGIN_EVT:
                case LINK_FAILOVER_END_EVT:
                        break;
                case LINK_SYNCH_END_EVT:
                default:
                        goto illegal_evt;
                }
                break;
        case LINK_ESTABLISHED:
                switch (evt) {
                case LINK_PEER_RESET_EVT:
                        l->state = LINK_PEER_RESET;
                        rc |= TIPC_LINK_DOWN_EVT;
                        break;
                case LINK_FAILURE_EVT:
                        l->state = LINK_RESETTING;
                        rc |= TIPC_LINK_DOWN_EVT;
                        break;
                case LINK_RESET_EVT:
                        l->state = LINK_RESET;
                        break;
                case LINK_ESTABLISH_EVT:
                case LINK_SYNCH_END_EVT:
                        break;
                case LINK_SYNCH_BEGIN_EVT:
                        l->state = LINK_SYNCHING;
                        break;
                case LINK_FAILOVER_BEGIN_EVT:
                case LINK_FAILOVER_END_EVT:
                default:
                        goto illegal_evt;
                }
                break;
        case LINK_SYNCHING:
                switch (evt) {
                case LINK_PEER_RESET_EVT:
                        l->state = LINK_PEER_RESET;
                        rc |= TIPC_LINK_DOWN_EVT;
                        break;
                case LINK_FAILURE_EVT:
                        l->state = LINK_RESETTING;
                        rc |= TIPC_LINK_DOWN_EVT;
                        break;
                case LINK_RESET_EVT:
                        l->state = LINK_RESET;
                        break;
                case LINK_ESTABLISH_EVT:
                case LINK_SYNCH_BEGIN_EVT:
                        break;
                case LINK_SYNCH_END_EVT:
                        l->state = LINK_ESTABLISHED;
                        break;
                case LINK_FAILOVER_BEGIN_EVT:
                case LINK_FAILOVER_END_EVT:
                default:
                        goto illegal_evt;
                }
                break;
        default:
                pr_err("Unknown FSM state %x in %s\n", l->state, l->name);
        }
        trace_tipc_link_fsm(l->name, old_state, l->state, evt);
        return rc;
illegal_evt:
        pr_err("Illegal FSM event %x in state %x on link %s\n",
               evt, l->state, l->name);
        trace_tipc_link_fsm(l->name, old_state, l->state, evt);
        return rc;
}

/* link_profile_stats - update statistical profiling of traffic
 */
static void link_profile_stats(struct tipc_link *l)
{
        struct sk_buff *skb;
        struct tipc_msg *msg;
        int length;

        /* Update counters used in statistical profiling of send traffic */
        l->stats.accu_queue_sz += skb_queue_len(&l->transmq);
        l->stats.queue_sz_counts++;

        skb = skb_peek(&l->transmq);
        if (!skb)
                return;
        msg = buf_msg(skb);
        length = msg_size(msg);

        if (msg_user(msg) == MSG_FRAGMENTER) {
                if (msg_type(msg) != FIRST_FRAGMENT)
                        return;
                length = msg_size(msg_inner_hdr(msg));
        }
        l->stats.msg_lengths_total += length;
        l->stats.msg_length_counts++;
        if (length <= 64)
                l->stats.msg_length_profile[0]++;
        else if (length <= 256)
                l->stats.msg_length_profile[1]++;
        else if (length <= 1024)
                l->stats.msg_length_profile[2]++;
        else if (length <= 4096)
                l->stats.msg_length_profile[3]++;
        else if (length <= 16384)
                l->stats.msg_length_profile[4]++;
        else if (length <= 32768)
                l->stats.msg_length_profile[5]++;
        else
                l->stats.msg_length_profile[6]++;
}

/**
 * tipc_link_too_silent - check if link is "too silent"
 * @l: tipc link to be checked
 *
 * Return: true if the link 'silent_intv_cnt' is about to reach the
 * 'abort_limit' value, otherwise false
 */
bool tipc_link_too_silent(struct tipc_link *l)
{
        return (l->silent_intv_cnt + 2 > l->abort_limit);
}

/* tipc_link_timeout - perform periodic task as instructed from node timeout
 */
int tipc_link_timeout(struct tipc_link *l, struct sk_buff_head *xmitq)
{
        int mtyp = 0;
        int rc = 0;
        bool state = false;
        bool probe = false;
        bool setup = false;
        u16 bc_snt = l->bc_sndlink->snd_nxt - 1;
        u16 bc_acked = l->bc_rcvlink->acked;
        struct tipc_mon_state *mstate = &l->mon_state;

        trace_tipc_link_timeout(l, TIPC_DUMP_NONE, " ");
        trace_tipc_link_too_silent(l, TIPC_DUMP_ALL, " ");
        switch (l->state) {
        case LINK_ESTABLISHED:
        case LINK_SYNCHING:
                mtyp = STATE_MSG;
                link_profile_stats(l);
                tipc_mon_get_state(l->net, l->addr, mstate, l->bearer_id);
                if (mstate->reset || (l->silent_intv_cnt > l->abort_limit))
                        return tipc_link_fsm_evt(l, LINK_FAILURE_EVT);
                state = bc_acked != bc_snt;
                state |= l->bc_rcvlink->rcv_unacked;
                state |= l->rcv_unacked;
                state |= !skb_queue_empty(&l->transmq);
                probe = mstate->probing;
                probe |= l->silent_intv_cnt;
                if (probe || mstate->monitoring)
                        l->silent_intv_cnt++;
                probe |= !skb_queue_empty(&l->deferdq);
                if (l->snd_nxt == l->checkpoint) {
                        tipc_link_update_cwin(l, 0, 0);
                        probe = true;
                }
                l->checkpoint = l->snd_nxt;
                break;
        case LINK_RESET:
                setup = l->rst_cnt++ <= 4;
                setup |= !(l->rst_cnt % 16);
                mtyp = RESET_MSG;
                break;
        case LINK_ESTABLISHING:
                setup = true;
                mtyp = ACTIVATE_MSG;
                break;
        case LINK_PEER_RESET:
        case LINK_RESETTING:
        case LINK_FAILINGOVER:
                break;
        default:
                break;
        }

        if (state || probe || setup)
                tipc_link_build_proto_msg(l, mtyp, probe, 0, 0, 0, 0, xmitq);

        return rc;
}

/**
 * link_schedule_user - schedule a message sender for wakeup after congestion
 * @l: congested link
 * @hdr: header of message that is being sent
 * Create pseudo msg to send back to user when congestion abates
 */
static int link_schedule_user(struct tipc_link *l, struct tipc_msg *hdr)
{
        u32 dnode = tipc_own_addr(l->net);
        u32 dport = msg_origport(hdr);
        struct sk_buff *skb;

        /* Create and schedule wakeup pseudo message */
        skb = tipc_msg_create(SOCK_WAKEUP, 0, INT_H_SIZE, 0,
                              dnode, l->addr, dport, 0, 0);
        if (!skb)
                return -ENOBUFS;
        msg_set_dest_droppable(buf_msg(skb), true);
        TIPC_SKB_CB(skb)->chain_imp = msg_importance(hdr);
        skb_queue_tail(&l->wakeupq, skb);
        l->stats.link_congs++;
        trace_tipc_link_conges(l, TIPC_DUMP_ALL, "wakeup scheduled!");
        return -ELINKCONG;
}

/**
 * link_prepare_wakeup - prepare users for wakeup after congestion
 * @l: congested link
 * Wake up a number of waiting users, as permitted by available space
 * in the send queue
 */
static void link_prepare_wakeup(struct tipc_link *l)
{
        struct sk_buff_head *wakeupq = &l->wakeupq;
        struct sk_buff_head *inputq = l->inputq;
        struct sk_buff *skb, *tmp;
        struct sk_buff_head tmpq;
        int avail[5] = {0,};
        int imp = 0;

        __skb_queue_head_init(&tmpq);

        for (; imp <= TIPC_SYSTEM_IMPORTANCE; imp++)
                avail[imp] = l->backlog[imp].limit - l->backlog[imp].len;

        skb_queue_walk_safe(wakeupq, skb, tmp) {
                imp = TIPC_SKB_CB(skb)->chain_imp;
                if (avail[imp] <= 0)
                        continue;
                avail[imp]--;
                __skb_unlink(skb, wakeupq);
                __skb_queue_tail(&tmpq, skb);
        }

        spin_lock_bh(&inputq->lock);
        skb_queue_splice_tail(&tmpq, inputq);
        spin_unlock_bh(&inputq->lock);

}

/**
 * tipc_link_set_skb_retransmit_time - set the time at which retransmission of
 *                                     the given skb should be next attempted
 * @skb: skb to set a future retransmission time for
 * @l: link the skb will be transmitted on
 */
static void tipc_link_set_skb_retransmit_time(struct sk_buff *skb,
                                              struct tipc_link *l)
{
        if (link_is_bc_sndlink(l))
                TIPC_SKB_CB(skb)->nxt_retr = TIPC_BC_RETR_LIM;
        else
                TIPC_SKB_CB(skb)->nxt_retr = TIPC_UC_RETR_TIME;
}

void tipc_link_reset(struct tipc_link *l)
{
        struct sk_buff_head list;
        u32 imp;

        __skb_queue_head_init(&list);

        l->in_session = false;
        /* Force re-synch of peer session number before establishing */
        l->peer_session--;
        l->session++;
        l->mtu = l->advertised_mtu;

        spin_lock_bh(&l->wakeupq.lock);
        skb_queue_splice_init(&l->wakeupq, &list);
        spin_unlock_bh(&l->wakeupq.lock);

        spin_lock_bh(&l->inputq->lock);
        skb_queue_splice_init(&list, l->inputq);
        spin_unlock_bh(&l->inputq->lock);

        __skb_queue_purge(&l->transmq);
        __skb_queue_purge(&l->deferdq);
        __skb_queue_purge(&l->backlogq);
        __skb_queue_purge(&l->failover_deferdq);
        for (imp = 0; imp <= TIPC_SYSTEM_IMPORTANCE; imp++) {
                l->backlog[imp].len = 0;
                l->backlog[imp].target_bskb = NULL;
        }
        kfree_skb(l->reasm_buf);
        kfree_skb(l->reasm_tnlmsg);
        kfree_skb(l->failover_reasm_skb);
        l->reasm_buf = NULL;
        l->reasm_tnlmsg = NULL;
        l->failover_reasm_skb = NULL;
        l->rcv_unacked = 0;
        l->snd_nxt = 1;
        l->rcv_nxt = 1;
        l->snd_nxt_state = 1;
        l->rcv_nxt_state = 1;
        l->acked = 0;
        l->last_gap = 0;
        kfree(l->last_ga);
        l->last_ga = NULL;
        l->silent_intv_cnt = 0;
        l->rst_cnt = 0;
        l->bc_peer_is_up = false;
        memset(&l->mon_state, 0, sizeof(l->mon_state));
        tipc_link_reset_stats(l);
}

/**
 * tipc_link_xmit(): enqueue buffer list according to queue situation
 * @l: link to use
 * @list: chain of buffers containing message
 * @xmitq: returned list of packets to be sent by caller
 *
 * Consumes the buffer chain.
 * Messages at TIPC_SYSTEM_IMPORTANCE are always accepted
 * Return: 0 if success, or errno: -ELINKCONG, -EMSGSIZE or -ENOBUFS
 */
int tipc_link_xmit(struct tipc_link *l, struct sk_buff_head *list,
                   struct sk_buff_head *xmitq)
{
        struct sk_buff_head *backlogq = &l->backlogq;
        struct sk_buff_head *transmq = &l->transmq;
        struct sk_buff *skb, *_skb;
        u16 bc_ack = l->bc_rcvlink->rcv_nxt - 1;
        u16 ack = l->rcv_nxt - 1;
        u16 seqno = l->snd_nxt;
        int pkt_cnt = skb_queue_len(list);
        unsigned int mss = tipc_link_mss(l);
        unsigned int cwin = l->window;
        unsigned int mtu = l->mtu;
        struct tipc_msg *hdr;
        bool new_bundle;
        int rc = 0;
        int imp;

        if (pkt_cnt <= 0)
                return 0;

        hdr = buf_msg(skb_peek(list));
        if (unlikely(msg_size(hdr) > mtu)) {
                pr_warn("Too large msg, purging xmit list %d %d %d %d %d!\n",
                        skb_queue_len(list), msg_user(hdr),
                        msg_type(hdr), msg_size(hdr), mtu);
                __skb_queue_purge(list);
                return -EMSGSIZE;
        }

        imp = msg_importance(hdr);
        /* Allow oversubscription of one data msg per source at congestion */
        if (unlikely(l->backlog[imp].len >= l->backlog[imp].limit)) {
                if (imp == TIPC_SYSTEM_IMPORTANCE) {
                        pr_warn("%s<%s>, link overflow", link_rst_msg, l->name);
                        __skb_queue_purge(list);
                        return -ENOBUFS;
                }
                rc = link_schedule_user(l, hdr);
        }

        if (pkt_cnt > 1) {
                l->stats.sent_fragmented++;
                l->stats.sent_fragments += pkt_cnt;
        }

        /* Prepare each packet for sending, and add to relevant queue: */
        while ((skb = __skb_dequeue(list))) {
                if (likely(skb_queue_len(transmq) < cwin)) {
                        hdr = buf_msg(skb);
                        msg_set_seqno(hdr, seqno);
                        msg_set_ack(hdr, ack);
                        msg_set_bcast_ack(hdr, bc_ack);
                        _skb = skb_clone(skb, GFP_ATOMIC);
                        if (!_skb) {
                                kfree_skb(skb);
                                __skb_queue_purge(list);
                                return -ENOBUFS;
                        }
                        __skb_queue_tail(transmq, skb);
                        tipc_link_set_skb_retransmit_time(skb, l);
                        __skb_queue_tail(xmitq, _skb);
                        TIPC_SKB_CB(skb)->ackers = l->ackers;
                        l->rcv_unacked = 0;
                        l->stats.sent_pkts++;
                        seqno++;
                        continue;
                }
                if (tipc_msg_try_bundle(l->backlog[imp].target_bskb, &skb,
                                        mss, l->addr, &new_bundle)) {
                        if (skb) {
                                /* Keep a ref. to the skb for next try */
                                l->backlog[imp].target_bskb = skb;
                                l->backlog[imp].len++;
                                __skb_queue_tail(backlogq, skb);
                        } else {
                                if (new_bundle) {
                                        l->stats.sent_bundles++;
                                        l->stats.sent_bundled++;
                                }
                                l->stats.sent_bundled++;
                        }
                        continue;
                }
                l->backlog[imp].target_bskb = NULL;
                l->backlog[imp].len += (1 + skb_queue_len(list));
                __skb_queue_tail(backlogq, skb);
                skb_queue_splice_tail_init(list, backlogq);
        }
        l->snd_nxt = seqno;
        return rc;
}

static void tipc_link_update_cwin(struct tipc_link *l, int released,
                                  bool retransmitted)
{
        int bklog_len = skb_queue_len(&l->backlogq);
        struct sk_buff_head *txq = &l->transmq;
        int txq_len = skb_queue_len(txq);
        u16 cwin = l->window;

        /* Enter fast recovery */
        if (unlikely(retransmitted)) {
                l->ssthresh = max_t(u16, l->window / 2, 300);
                l->window = min_t(u16, l->ssthresh, l->window);
                return;
        }
        /* Enter slow start */
        if (unlikely(!released)) {
                l->ssthresh = max_t(u16, l->window / 2, 300);
                l->window = l->min_win;
                return;
        }
        /* Don't increase window if no pressure on the transmit queue */
        if (txq_len + bklog_len < cwin)
                return;

        /* Don't increase window if there are holes the transmit queue */
        if (txq_len && l->snd_nxt - buf_seqno(skb_peek(txq)) != txq_len)
                return;

        l->cong_acks += released;

        /* Slow start  */
        if (cwin <= l->ssthresh) {
                l->window = min_t(u16, cwin + released, l->max_win);
                return;
        }
        /* Congestion avoidance */
        if (l->cong_acks < cwin)
                return;
        l->window = min_t(u16, ++cwin, l->max_win);
        l->cong_acks = 0;
}

static void tipc_link_advance_backlog(struct tipc_link *l,
                                      struct sk_buff_head *xmitq)
{
        u16 bc_ack = l->bc_rcvlink->rcv_nxt - 1;
        struct sk_buff_head *txq = &l->transmq;
        struct sk_buff *skb, *_skb;
        u16 ack = l->rcv_nxt - 1;
        u16 seqno = l->snd_nxt;
        struct tipc_msg *hdr;
        u16 cwin = l->window;
        u32 imp;

        while (skb_queue_len(txq) < cwin) {
                skb = skb_peek(&l->backlogq);
                if (!skb)
                        break;
                _skb = skb_clone(skb, GFP_ATOMIC);
                if (!_skb)
                        break;
                __skb_dequeue(&l->backlogq);
                hdr = buf_msg(skb);
                imp = msg_importance(hdr);
                l->backlog[imp].len--;
                if (unlikely(skb == l->backlog[imp].target_bskb))
                        l->backlog[imp].target_bskb = NULL;
                __skb_queue_tail(&l->transmq, skb);
                tipc_link_set_skb_retransmit_time(skb, l);

                __skb_queue_tail(xmitq, _skb);
                TIPC_SKB_CB(skb)->ackers = l->ackers;
                msg_set_seqno(hdr, seqno);
                msg_set_ack(hdr, ack);
                msg_set_bcast_ack(hdr, bc_ack);
                l->rcv_unacked = 0;
                l->stats.sent_pkts++;
                seqno++;
        }
        l->snd_nxt = seqno;
}

/**
 * link_retransmit_failure() - Detect repeated retransmit failures
 * @l: tipc link sender
 * @r: tipc link receiver (= l in case of unicast)
 * @rc: returned code
 *
 * Return: true if the repeated retransmit failures happens, otherwise
 * false
 */
static bool link_retransmit_failure(struct tipc_link *l, struct tipc_link *r,
                                    int *rc)
{
        struct sk_buff *skb = skb_peek(&l->transmq);
        struct tipc_msg *hdr;

        if (!skb)
                return false;

        if (!TIPC_SKB_CB(skb)->retr_cnt)
                return false;

        if (!time_after(jiffies, TIPC_SKB_CB(skb)->retr_stamp +
                        msecs_to_jiffies(r->tolerance * 10)))
                return false;

        hdr = buf_msg(skb);
        if (link_is_bc_sndlink(l) && !less(r->acked, msg_seqno(hdr)))
                return false;

        pr_warn("Retransmission failure on link <%s>\n", l->name);
        link_print(l, "State of link ");
        pr_info("Failed msg: usr %u, typ %u, len %u, err %u\n",
                msg_user(hdr), msg_type(hdr), msg_size(hdr), msg_errcode(hdr));
        pr_info("sqno %u, prev: %x, dest: %x\n",
                msg_seqno(hdr), msg_prevnode(hdr), msg_destnode(hdr));
        pr_info("retr_stamp %d, retr_cnt %d\n",
                jiffies_to_msecs(TIPC_SKB_CB(skb)->retr_stamp),
                TIPC_SKB_CB(skb)->retr_cnt);

        trace_tipc_list_dump(&l->transmq, true, "retrans failure!");
        trace_tipc_link_dump(l, TIPC_DUMP_NONE, "retrans failure!");
        trace_tipc_link_dump(r, TIPC_DUMP_NONE, "retrans failure!");

        if (link_is_bc_sndlink(l)) {
                r->state = LINK_RESET;
                *rc |= TIPC_LINK_DOWN_EVT;
        } else {
                *rc |= tipc_link_fsm_evt(l, LINK_FAILURE_EVT);
        }

        return true;
}

/* tipc_data_input - deliver data and name distr msgs to upper layer
 *
 * Consumes buffer if message is of right type
 * Node lock must be held
 */
static bool tipc_data_input(struct tipc_link *l, struct sk_buff *skb,
                            struct sk_buff_head *inputq)
{
        struct sk_buff_head *mc_inputq = l->bc_rcvlink->inputq;
        struct tipc_msg *hdr = buf_msg(skb);

        switch (msg_user(hdr)) {
        case TIPC_LOW_IMPORTANCE:
        case TIPC_MEDIUM_IMPORTANCE:
        case TIPC_HIGH_IMPORTANCE:
        case TIPC_CRITICAL_IMPORTANCE:
                if (unlikely(msg_in_group(hdr) || msg_mcast(hdr))) {
                        skb_queue_tail(mc_inputq, skb);
                        return true;
                }
                fallthrough;
        case CONN_MANAGER:
                skb_queue_tail(inputq, skb);
                return true;
        case GROUP_PROTOCOL:
                skb_queue_tail(mc_inputq, skb);
                return true;
        case NAME_DISTRIBUTOR:
                l->bc_rcvlink->state = LINK_ESTABLISHED;
                skb_queue_tail(l->namedq, skb);
                return true;
        case MSG_BUNDLER:
        case TUNNEL_PROTOCOL:
        case MSG_FRAGMENTER:
        case BCAST_PROTOCOL:
                return false;
#ifdef CONFIG_TIPC_CRYPTO
        case MSG_CRYPTO:
                if (sysctl_tipc_key_exchange_enabled &&
                    TIPC_SKB_CB(skb)->decrypted) {
                        tipc_crypto_msg_rcv(l->net, skb);
                        return true;
                }
                fallthrough;
#endif
        default:
                pr_warn("Dropping received illegal msg type\n");
                kfree_skb(skb);
                return true;
        }
}

/* tipc_link_input - process packet that has passed link protocol check
 *
 * Consumes buffer
 */
static int tipc_link_input(struct tipc_link *l, struct sk_buff *skb,
                           struct sk_buff_head *inputq,
                           struct sk_buff **reasm_skb)
{
        struct tipc_msg *hdr = buf_msg(skb);
        struct sk_buff *iskb;
        struct sk_buff_head tmpq;
        int usr = msg_user(hdr);
        int pos = 0;

        if (usr == MSG_BUNDLER) {
                skb_queue_head_init(&tmpq);
                l->stats.recv_bundles++;
                l->stats.recv_bundled += msg_msgcnt(hdr);
                while (tipc_msg_extract(skb, &iskb, &pos))
                        tipc_data_input(l, iskb, &tmpq);
                tipc_skb_queue_splice_tail(&tmpq, inputq);
                return 0;
        } else if (usr == MSG_FRAGMENTER) {
                l->stats.recv_fragments++;
                if (tipc_buf_append(reasm_skb, &skb)) {
                        l->stats.recv_fragmented++;
                        tipc_data_input(l, skb, inputq);
                } else if (!*reasm_skb && !link_is_bc_rcvlink(l)) {
                        pr_warn_ratelimited("Unable to build fragment list\n");
                        return tipc_link_fsm_evt(l, LINK_FAILURE_EVT);
                }
                return 0;
        } else if (usr == BCAST_PROTOCOL) {
                tipc_bcast_lock(l->net);
                tipc_link_bc_init_rcv(l->bc_rcvlink, hdr);
                tipc_bcast_unlock(l->net);
        }

        kfree_skb(skb);
        return 0;
}

/* tipc_link_tnl_rcv() - receive TUNNEL_PROTOCOL message, drop or process the
 *                         inner message along with the ones in the old link's
 *                         deferdq
 * @l: tunnel link
 * @skb: TUNNEL_PROTOCOL message
 * @inputq: queue to put messages ready for delivery
 */
static int tipc_link_tnl_rcv(struct tipc_link *l, struct sk_buff *skb,
                             struct sk_buff_head *inputq)
{
        struct sk_buff **reasm_skb = &l->failover_reasm_skb;
        struct sk_buff **reasm_tnlmsg = &l->reasm_tnlmsg;
        struct sk_buff_head *fdefq = &l->failover_deferdq;
        struct tipc_msg *hdr = buf_msg(skb);
        struct sk_buff *iskb;
        int ipos = 0;
        int rc = 0;
        u16 seqno;

        if (msg_type(hdr) == SYNCH_MSG) {
                kfree_skb(skb);
                return 0;
        }

        /* Not a fragment? */
        if (likely(!msg_nof_fragms(hdr))) {
                if (unlikely(!tipc_msg_extract(skb, &iskb, &ipos))) {
                        pr_warn_ratelimited("Unable to extract msg, defq: %d\n",
                                            skb_queue_len(fdefq));
                        return 0;
                }
                kfree_skb(skb);
        } else {
                /* Set fragment type for buf_append */
                if (msg_fragm_no(hdr) == 1)
                        msg_set_type(hdr, FIRST_FRAGMENT);
                else if (msg_fragm_no(hdr) < msg_nof_fragms(hdr))
                        msg_set_type(hdr, FRAGMENT);
                else
                        msg_set_type(hdr, LAST_FRAGMENT);

                if (!tipc_buf_append(reasm_tnlmsg, &skb)) {
                        /* Successful but non-complete reassembly? */
                        if (*reasm_tnlmsg || link_is_bc_rcvlink(l))
                                return 0;
                        pr_warn_ratelimited("Unable to reassemble tunnel msg\n");
                        return tipc_link_fsm_evt(l, LINK_FAILURE_EVT);
                }
                iskb = skb;
        }

        do {
                seqno = buf_seqno(iskb);
                if (unlikely(less(seqno, l->drop_point))) {
                        kfree_skb(iskb);
                        continue;
                }
                if (unlikely(seqno != l->drop_point)) {
                        __tipc_skb_queue_sorted(fdefq, seqno, iskb);
                        continue;
                }

                l->drop_point++;
                if (!tipc_data_input(l, iskb, inputq))
                        rc |= tipc_link_input(l, iskb, inputq, reasm_skb);
                if (unlikely(rc))
                        break;
        } while ((iskb = __tipc_skb_dequeue(fdefq, l->drop_point)));

        return rc;
}

/**
 * tipc_get_gap_ack_blks - get Gap ACK blocks from PROTOCOL/STATE_MSG
 * @ga: returned pointer to the Gap ACK blocks if any
 * @l: the tipc link
 * @hdr: the PROTOCOL/STATE_MSG header
 * @uc: desired Gap ACK blocks type, i.e. unicast (= 1) or broadcast (= 0)
 *
 * Return: the total Gap ACK blocks size
 */
u16 tipc_get_gap_ack_blks(struct tipc_gap_ack_blks **ga, struct tipc_link *l,
                          struct tipc_msg *hdr, bool uc)
{
        struct tipc_gap_ack_blks *p;
        u16 sz = 0;

        /* Does peer support the Gap ACK blocks feature? */
        if (l->peer_caps & TIPC_GAP_ACK_BLOCK) {
                p = (struct tipc_gap_ack_blks *)msg_data(hdr);
                sz = ntohs(p->len);
                /* Sanity check */
                if (sz == struct_size(p, gacks, size_add(p->ugack_cnt, p->bgack_cnt))) {
                        /* Good, check if the desired type exists */
                        if ((uc && p->ugack_cnt) || (!uc && p->bgack_cnt))
                                goto ok;
                /* Backward compatible: peer might not support bc, but uc? */
                } else if (uc && sz == struct_size(p, gacks, p->ugack_cnt)) {
                        if (p->ugack_cnt) {
                                p->bgack_cnt = 0;
                                goto ok;
                        }
                }
        }
        /* Other cases: ignore! */
        p = NULL;

ok:
        *ga = p;
        return sz;
}

static u8 __tipc_build_gap_ack_blks(struct tipc_gap_ack_blks *ga,
                                    struct tipc_link *l, u8 start_index)
{
        struct tipc_gap_ack *gacks = &ga->gacks[start_index];
        struct sk_buff *skb = skb_peek(&l->deferdq);
        u16 expect, seqno = 0;
        u8 n = 0;

        if (!skb)
                return 0;

        expect = buf_seqno(skb);
        skb_queue_walk(&l->deferdq, skb) {
                seqno = buf_seqno(skb);
                if (unlikely(more(seqno, expect))) {
                        gacks[n].ack = htons(expect - 1);
                        gacks[n].gap = htons(seqno - expect);
                        if (++n >= MAX_GAP_ACK_BLKS / 2) {
                                pr_info_ratelimited("Gacks on %s: %d, ql: %d!\n",
                                                    l->name, n,
                                                    skb_queue_len(&l->deferdq));
                                return n;
                        }
                } else if (unlikely(less(seqno, expect))) {
                        pr_warn("Unexpected skb in deferdq!\n");
                        continue;
                }
                expect = seqno + 1;
        }

        /* last block */
        gacks[n].ack = htons(seqno);
        gacks[n].gap = 0;
        n++;
        return n;
}

/* tipc_build_gap_ack_blks - build Gap ACK blocks
 * @l: tipc unicast link
 * @hdr: the tipc message buffer to store the Gap ACK blocks after built
 *
 * The function builds Gap ACK blocks for both the unicast & broadcast receiver
 * links of a certain peer, the buffer after built has the network data format
 * as found at the struct tipc_gap_ack_blks definition.
 *
 * returns the actual allocated memory size
 */
static u16 tipc_build_gap_ack_blks(struct tipc_link *l, struct tipc_msg *hdr)
{
        struct tipc_link *bcl = l->bc_rcvlink;
        struct tipc_gap_ack_blks *ga;
        u16 len;

        ga = (struct tipc_gap_ack_blks *)msg_data(hdr);

        /* Start with broadcast link first */
        tipc_bcast_lock(bcl->net);
        msg_set_bcast_ack(hdr, bcl->rcv_nxt - 1);
        msg_set_bc_gap(hdr, link_bc_rcv_gap(bcl));
        ga->bgack_cnt = __tipc_build_gap_ack_blks(ga, bcl, 0);
        tipc_bcast_unlock(bcl->net);

        /* Now for unicast link, but an explicit NACK only (???) */
        ga->ugack_cnt = (msg_seq_gap(hdr)) ?
                        __tipc_build_gap_ack_blks(ga, l, ga->bgack_cnt) : 0;

        /* Total len */
        len = struct_size(ga, gacks, size_add(ga->bgack_cnt, ga->ugack_cnt));
        ga->len = htons(len);
        return len;
}

/* tipc_link_advance_transmq - advance TIPC link transmq queue by releasing
 *                               acked packets, also doing retransmissions if
 *                               gaps found
 * @l: tipc link with transmq queue to be advanced
 * @r: tipc link "receiver" i.e. in case of broadcast (= "l" if unicast)
 * @acked: seqno of last packet acked by peer without any gaps before
 * @gap: # of gap packets
 * @ga: buffer pointer to Gap ACK blocks from peer
 * @xmitq: queue for accumulating the retransmitted packets if any
 * @retransmitted: returned boolean value if a retransmission is really issued
 * @rc: returned code e.g. TIPC_LINK_DOWN_EVT if a repeated retransmit failures
 *      happens (- unlikely case)
 *
 * Return: the number of packets released from the link transmq
 */
static int tipc_link_advance_transmq(struct tipc_link *l, struct tipc_link *r,
                                     u16 acked, u16 gap,
                                     struct tipc_gap_ack_blks *ga,
                                     struct sk_buff_head *xmitq,
                                     bool *retransmitted, int *rc)
{
        struct tipc_gap_ack_blks *last_ga = r->last_ga, *this_ga = NULL;
        struct tipc_gap_ack *gacks = NULL;
        struct sk_buff *skb, *_skb, *tmp;
        struct tipc_msg *hdr;
        u32 qlen = skb_queue_len(&l->transmq);
        u16 nacked = acked, ngap = gap, gack_cnt = 0;
        u16 bc_ack = l->bc_rcvlink->rcv_nxt - 1;
        u16 ack = l->rcv_nxt - 1;
        u16 seqno, n = 0;
        u16 end = r->acked, start = end, offset = r->last_gap;
        u16 si = (last_ga) ? last_ga->start_index : 0;
        bool is_uc = !link_is_bc_sndlink(l);
        bool bc_has_acked = false;

        trace_tipc_link_retrans(r, acked + 1, acked + gap, &l->transmq);

        /* Determine Gap ACK blocks if any for the particular link */
        if (ga && is_uc) {
                /* Get the Gap ACKs, uc part */
                gack_cnt = ga->ugack_cnt;
                gacks = &ga->gacks[ga->bgack_cnt];
        } else if (ga) {
                /* Copy the Gap ACKs, bc part, for later renewal if needed */
                this_ga = kmemdup(ga, struct_size(ga, gacks, ga->bgack_cnt),
                                  GFP_ATOMIC);
                if (likely(this_ga)) {
                        this_ga->start_index = 0;
                        /* Start with the bc Gap ACKs */
                        gack_cnt = this_ga->bgack_cnt;
                        gacks = &this_ga->gacks[0];
                } else {
                        /* Hmm, we can get in trouble..., simply ignore it */
                        pr_warn_ratelimited("Ignoring bc Gap ACKs, no memory\n");
                }
        }

        /* Advance the link transmq */
        skb_queue_walk_safe(&l->transmq, skb, tmp) {
                seqno = buf_seqno(skb);

next_gap_ack:
                if (less_eq(seqno, nacked)) {
                        if (is_uc)
                                goto release;
                        /* Skip packets peer has already acked */
                        if (!more(seqno, r->acked))
                                continue;
                        /* Get the next of last Gap ACK blocks */
                        while (more(seqno, end)) {
                                if (!last_ga || si >= last_ga->bgack_cnt)
                                        break;
                                start = end + offset + 1;
                                end = ntohs(last_ga->gacks[si].ack);
                                offset = ntohs(last_ga->gacks[si].gap);
                                si++;
                                WARN_ONCE(more(start, end) ||
                                          (!offset &&
                                           si < last_ga->bgack_cnt) ||
                                          si > MAX_GAP_ACK_BLKS,
                                          "Corrupted Gap ACK: %d %d %d %d %d\n",
                                          start, end, offset, si,
                                          last_ga->bgack_cnt);
                        }
                        /* Check against the last Gap ACK block */
                        if (tipc_in_range(seqno, start, end))
                                continue;
                        /* Update/release the packet peer is acking */
                        bc_has_acked = true;
                        if (--TIPC_SKB_CB(skb)->ackers)
                                continue;
release:
                        /* release skb */
                        __skb_unlink(skb, &l->transmq);
                        kfree_skb(skb);
                } else if (less_eq(seqno, nacked + ngap)) {
                        /* First gap: check if repeated retrans failures? */
                        if (unlikely(seqno == acked + 1 &&
                                     link_retransmit_failure(l, r, rc))) {
                                /* Ignore this bc Gap ACKs if any */
                                kfree(this_ga);
                                this_ga = NULL;
                                break;
                        }
                        /* retransmit skb if unrestricted*/
                        if (time_before(jiffies, TIPC_SKB_CB(skb)->nxt_retr))
                                continue;
                        tipc_link_set_skb_retransmit_time(skb, l);
                        _skb = pskb_copy(skb, GFP_ATOMIC);
                        if (!_skb)
                                continue;
                        hdr = buf_msg(_skb);
                        msg_set_ack(hdr, ack);
                        msg_set_bcast_ack(hdr, bc_ack);
                        _skb->priority = TC_PRIO_CONTROL;
                        __skb_queue_tail(xmitq, _skb);
                        l->stats.retransmitted++;
                        if (!is_uc)
                                r->stats.retransmitted++;
                        *retransmitted = true;
                        /* Increase actual retrans counter & mark first time */
                        if (!TIPC_SKB_CB(skb)->retr_cnt++)
                                TIPC_SKB_CB(skb)->retr_stamp = jiffies;
                } else {
                        /* retry with Gap ACK blocks if any */
                        if (n >= gack_cnt)
                                break;
                        nacked = ntohs(gacks[n].ack);
                        ngap = ntohs(gacks[n].gap);
                        n++;
                        goto next_gap_ack;
                }
        }

        /* Renew last Gap ACK blocks for bc if needed */
        if (bc_has_acked) {
                if (this_ga) {
                        kfree(last_ga);
                        r->last_ga = this_ga;
                        r->last_gap = gap;
                } else if (last_ga) {
                        if (less(acked, start)) {
                                si--;
                                offset = start - acked - 1;
                        } else if (less(acked, end)) {
                                acked = end;
                        }
                        if (si < last_ga->bgack_cnt) {
                                last_ga->start_index = si;
                                r->last_gap = offset;
                        } else {
                                kfree(last_ga);
                                r->last_ga = NULL;
                                r->last_gap = 0;
                        }
                } else {
                        r->last_gap = 0;
                }
                r->acked = acked;
        } else {
                kfree(this_ga);
        }

        return qlen - skb_queue_len(&l->transmq);
}

/* tipc_link_build_state_msg: prepare link state message for transmission
 *
 * Note that sending of broadcast ack is coordinated among nodes, to reduce
 * risk of ack storms towards the sender
 */
int tipc_link_build_state_msg(struct tipc_link *l, struct sk_buff_head *xmitq)
{
        if (!l)
                return 0;

        /* Broadcast ACK must be sent via a unicast link => defer to caller */
        if (link_is_bc_rcvlink(l)) {
                if (((l->rcv_nxt ^ tipc_own_addr(l->net)) & 0xf) != 0xf)
                        return 0;
                l->rcv_unacked = 0;

                /* Use snd_nxt to store peer's snd_nxt in broadcast rcv link */
                l->snd_nxt = l->rcv_nxt;
                return TIPC_LINK_SND_STATE;
        }
        /* Unicast ACK */
        l->rcv_unacked = 0;
        l->stats.sent_acks++;
        tipc_link_build_proto_msg(l, STATE_MSG, 0, 0, 0, 0, 0, xmitq);
        return 0;
}

/* tipc_link_build_reset_msg: prepare link RESET or ACTIVATE message
 */
void tipc_link_build_reset_msg(struct tipc_link *l, struct sk_buff_head *xmitq)
{
        int mtyp = RESET_MSG;
        struct sk_buff *skb;

        if (l->state == LINK_ESTABLISHING)
                mtyp = ACTIVATE_MSG;

        tipc_link_build_proto_msg(l, mtyp, 0, 0, 0, 0, 0, xmitq);

        /* Inform peer that this endpoint is going down if applicable */
        skb = skb_peek_tail(xmitq);
        if (skb && (l->state == LINK_RESET))
                msg_set_peer_stopping(buf_msg(skb), 1);
}

/* tipc_link_build_nack_msg: prepare link nack message for transmission
 * Note that sending of broadcast NACK is coordinated among nodes, to
 * reduce the risk of NACK storms towards the sender
 */
static int tipc_link_build_nack_msg(struct tipc_link *l,
                                    struct sk_buff_head *xmitq)
{
        u32 def_cnt = ++l->stats.deferred_recv;
        struct sk_buff_head *dfq = &l->deferdq;
        u32 defq_len = skb_queue_len(dfq);
        int match1, match2;

        if (link_is_bc_rcvlink(l)) {
                match1 = def_cnt & 0xf;
                match2 = tipc_own_addr(l->net) & 0xf;
                if (match1 == match2)
                        return TIPC_LINK_SND_STATE;
                return 0;
        }

        if (defq_len >= 3 && !((defq_len - 3) % 16)) {
                u16 rcvgap = buf_seqno(skb_peek(dfq)) - l->rcv_nxt;

                tipc_link_build_proto_msg(l, STATE_MSG, 0, 0,
                                          rcvgap, 0, 0, xmitq);
        }
        return 0;
}

/* tipc_link_rcv - process TIPC packets/messages arriving from off-node
 * @l: the link that should handle the message
 * @skb: TIPC packet
 * @xmitq: queue to place packets to be sent after this call
 */
int tipc_link_rcv(struct tipc_link *l, struct sk_buff *skb,
                  struct sk_buff_head *xmitq)
{
        struct sk_buff_head *defq = &l->deferdq;
        struct tipc_msg *hdr = buf_msg(skb);
        u16 seqno, rcv_nxt, win_lim;
        int released = 0;
        int rc = 0;

        /* Verify and update link state */
        if (unlikely(msg_user(hdr) == LINK_PROTOCOL))
                return tipc_link_proto_rcv(l, skb, xmitq);

        /* Don't send probe at next timeout expiration */
        l->silent_intv_cnt = 0;

        do {
                hdr = buf_msg(skb);
                seqno = msg_seqno(hdr);
                rcv_nxt = l->rcv_nxt;
                win_lim = rcv_nxt + TIPC_MAX_LINK_WIN;

                if (unlikely(!tipc_link_is_up(l))) {
                        if (l->state == LINK_ESTABLISHING)
                                rc = TIPC_LINK_UP_EVT;
                        kfree_skb(skb);
                        break;
                }

                /* Drop if outside receive window */
                if (unlikely(less(seqno, rcv_nxt) || more(seqno, win_lim))) {
                        l->stats.duplicates++;
                        kfree_skb(skb);
                        break;
                }
                released += tipc_link_advance_transmq(l, l, msg_ack(hdr), 0,
                                                      NULL, NULL, NULL, NULL);

                /* Defer delivery if sequence gap */
                if (unlikely(seqno != rcv_nxt)) {
                        if (!__tipc_skb_queue_sorted(defq, seqno, skb))
                                l->stats.duplicates++;
                        rc |= tipc_link_build_nack_msg(l, xmitq);
                        break;
                }

                /* Deliver packet */
                l->rcv_nxt++;
                l->stats.recv_pkts++;

                if (unlikely(msg_user(hdr) == TUNNEL_PROTOCOL))
                        rc |= tipc_link_tnl_rcv(l, skb, l->inputq);
                else if (!tipc_data_input(l, skb, l->inputq))
                        rc |= tipc_link_input(l, skb, l->inputq, &l->reasm_buf);
                if (unlikely(++l->rcv_unacked >= TIPC_MIN_LINK_WIN))
                        rc |= tipc_link_build_state_msg(l, xmitq);
                if (unlikely(rc & ~TIPC_LINK_SND_STATE))
                        break;
        } while ((skb = __tipc_skb_dequeue(defq, l->rcv_nxt)));

        /* Forward queues and wake up waiting users */
        if (released) {
                tipc_link_update_cwin(l, released, 0);
                tipc_link_advance_backlog(l, xmitq);
                if (unlikely(!skb_queue_empty(&l->wakeupq)))
                        link_prepare_wakeup(l);
        }
        return rc;
}

static void tipc_link_build_proto_msg(struct tipc_link *l, int mtyp, bool probe,
                                      bool probe_reply, u16 rcvgap,
                                      int tolerance, int priority,
                                      struct sk_buff_head *xmitq)
{
        struct tipc_mon_state *mstate = &l->mon_state;
        struct sk_buff_head *dfq = &l->deferdq;
        struct tipc_link *bcl = l->bc_rcvlink;
        struct tipc_msg *hdr;
        struct sk_buff *skb;
        bool node_up = tipc_link_is_up(bcl);
        u16 glen = 0, bc_rcvgap = 0;
        int dlen = 0;
        void *data;

        /* Don't send protocol message during reset or link failover */
        if (tipc_link_is_blocked(l))
                return;

        if (!tipc_link_is_up(l) && (mtyp == STATE_MSG))
                return;

        if ((probe || probe_reply) && !skb_queue_empty(dfq))
                rcvgap = buf_seqno(skb_peek(dfq)) - l->rcv_nxt;

        skb = tipc_msg_create(LINK_PROTOCOL, mtyp, INT_H_SIZE,
                              tipc_max_domain_size + MAX_GAP_ACK_BLKS_SZ,
                              l->addr, tipc_own_addr(l->net), 0, 0, 0);
        if (!skb)
                return;

        hdr = buf_msg(skb);
        data = msg_data(hdr);
        msg_set_session(hdr, l->session);
        msg_set_bearer_id(hdr, l->bearer_id);
        msg_set_net_plane(hdr, l->net_plane);
        msg_set_next_sent(hdr, l->snd_nxt);
        msg_set_ack(hdr, l->rcv_nxt - 1);
        msg_set_bcast_ack(hdr, bcl->rcv_nxt - 1);
        msg_set_bc_ack_invalid(hdr, !node_up);
        msg_set_last_bcast(hdr, l->bc_sndlink->snd_nxt - 1);
        msg_set_link_tolerance(hdr, tolerance);
        msg_set_linkprio(hdr, priority);
        msg_set_redundant_link(hdr, node_up);
        msg_set_seq_gap(hdr, 0);
        msg_set_seqno(hdr, l->snd_nxt + U16_MAX / 2);

        if (mtyp == STATE_MSG) {
                if (l->peer_caps & TIPC_LINK_PROTO_SEQNO)
                        msg_set_seqno(hdr, l->snd_nxt_state++);
                msg_set_seq_gap(hdr, rcvgap);
                bc_rcvgap = link_bc_rcv_gap(bcl);
                msg_set_bc_gap(hdr, bc_rcvgap);
                msg_set_probe(hdr, probe);
                msg_set_is_keepalive(hdr, probe || probe_reply);
                if (l->peer_caps & TIPC_GAP_ACK_BLOCK)
                        glen = tipc_build_gap_ack_blks(l, hdr);
                tipc_mon_prep(l->net, data + glen, &dlen, mstate, l->bearer_id);
                msg_set_size(hdr, INT_H_SIZE + glen + dlen);
                skb_trim(skb, INT_H_SIZE + glen + dlen);
                l->stats.sent_states++;
                l->rcv_unacked = 0;
        } else {
                /* RESET_MSG or ACTIVATE_MSG */
                if (mtyp == ACTIVATE_MSG) {
                        msg_set_dest_session_valid(hdr, 1);
                        msg_set_dest_session(hdr, l->peer_session);
                }
                msg_set_max_pkt(hdr, l->advertised_mtu);
                strcpy(data, l->if_name);
                msg_set_size(hdr, INT_H_SIZE + TIPC_MAX_IF_NAME);
                skb_trim(skb, INT_H_SIZE + TIPC_MAX_IF_NAME);
        }
        if (probe)
                l->stats.sent_probes++;
        if (rcvgap)
                l->stats.sent_nacks++;
        if (bc_rcvgap)
                bcl->stats.sent_nacks++;
        skb->priority = TC_PRIO_CONTROL;
        __skb_queue_tail(xmitq, skb);
        trace_tipc_proto_build(skb, false, l->name);
}

void tipc_link_create_dummy_tnl_msg(struct tipc_link *l,
                                    struct sk_buff_head *xmitq)
{
        u32 onode = tipc_own_addr(l->net);
        struct tipc_msg *hdr, *ihdr;
        struct sk_buff_head tnlq;
        struct sk_buff *skb;
        u32 dnode = l->addr;

        __skb_queue_head_init(&tnlq);
        skb = tipc_msg_create(TUNNEL_PROTOCOL, FAILOVER_MSG,
                              INT_H_SIZE, BASIC_H_SIZE,
                              dnode, onode, 0, 0, 0);
        if (!skb) {
                pr_warn("%sunable to create tunnel packet\n", link_co_err);
                return;
        }

        hdr = buf_msg(skb);
        msg_set_msgcnt(hdr, 1);
        msg_set_bearer_id(hdr, l->peer_bearer_id);

        ihdr = (struct tipc_msg *)msg_data(hdr);
        tipc_msg_init(onode, ihdr, TIPC_LOW_IMPORTANCE, TIPC_DIRECT_MSG,
                      BASIC_H_SIZE, dnode);
        msg_set_errcode(ihdr, TIPC_ERR_NO_PORT);
        __skb_queue_tail(&tnlq, skb);
        tipc_link_xmit(l, &tnlq, xmitq);
}

/* tipc_link_tnl_prepare(): prepare and return a list of tunnel packets
 * with contents of the link's transmit and backlog queues.
 */
void tipc_link_tnl_prepare(struct tipc_link *l, struct tipc_link *tnl,
                           int mtyp, struct sk_buff_head *xmitq)
{
        struct sk_buff *skb, *tnlskb;
        struct tipc_msg *hdr, tnlhdr;
        struct sk_buff_head *queue = &l->transmq;
        struct sk_buff_head tmpxq, tnlq, frags;
        u16 pktlen, pktcnt, seqno = l->snd_nxt;
        bool pktcnt_need_update = false;
        u16 syncpt;
        int rc;

        if (!tnl)
                return;

        __skb_queue_head_init(&tnlq);
        /* Link Synching:
         * From now on, send only one single ("dummy") SYNCH message
         * to peer. The SYNCH message does not contain any data, just
         * a header conveying the synch point to the peer.
         */
        if (mtyp == SYNCH_MSG && (tnl->peer_caps & TIPC_TUNNEL_ENHANCED)) {
                tnlskb = tipc_msg_create(TUNNEL_PROTOCOL, SYNCH_MSG,
                                         INT_H_SIZE, 0, l->addr,
                                         tipc_own_addr(l->net),
                                         0, 0, 0);
                if (!tnlskb) {
                        pr_warn("%sunable to create dummy SYNCH_MSG\n",
                                link_co_err);
                        return;
                }

                hdr = buf_msg(tnlskb);
                syncpt = l->snd_nxt + skb_queue_len(&l->backlogq) - 1;
                msg_set_syncpt(hdr, syncpt);
                msg_set_bearer_id(hdr, l->peer_bearer_id);
                __skb_queue_tail(&tnlq, tnlskb);
                tipc_link_xmit(tnl, &tnlq, xmitq);
                return;
        }

        __skb_queue_head_init(&tmpxq);
        __skb_queue_head_init(&frags);
        /* At least one packet required for safe algorithm => add dummy */
        skb = tipc_msg_create(TIPC_LOW_IMPORTANCE, TIPC_DIRECT_MSG,
                              BASIC_H_SIZE, 0, l->addr, tipc_own_addr(l->net),
                              0, 0, TIPC_ERR_NO_PORT);
        if (!skb) {
                pr_warn("%sunable to create tunnel packet\n", link_co_err);
                return;
        }
        __skb_queue_tail(&tnlq, skb);
        tipc_link_xmit(l, &tnlq, &tmpxq);
        __skb_queue_purge(&tmpxq);

        /* Initialize reusable tunnel packet header */
        tipc_msg_init(tipc_own_addr(l->net), &tnlhdr, TUNNEL_PROTOCOL,
                      mtyp, INT_H_SIZE, l->addr);
        if (mtyp == SYNCH_MSG)
                pktcnt = l->snd_nxt - buf_seqno(skb_peek(&l->transmq));
        else
                pktcnt = skb_queue_len(&l->transmq);
        pktcnt += skb_queue_len(&l->backlogq);
        msg_set_msgcnt(&tnlhdr, pktcnt);
        msg_set_bearer_id(&tnlhdr, l->peer_bearer_id);
tnl:
        /* Wrap each packet into a tunnel packet */
        skb_queue_walk(queue, skb) {
                hdr = buf_msg(skb);
                if (queue == &l->backlogq)
                        msg_set_seqno(hdr, seqno++);
                pktlen = msg_size(hdr);

                /* Tunnel link MTU is not large enough? This could be
                 * due to:
                 * 1) Link MTU has just changed or set differently;
                 * 2) Or FAILOVER on the top of a SYNCH message
                 *
                 * The 2nd case should not happen if peer supports
                 * TIPC_TUNNEL_ENHANCED
                 */
                if (pktlen > tnl->mtu - INT_H_SIZE) {
                        if (mtyp == FAILOVER_MSG &&
                            (tnl->peer_caps & TIPC_TUNNEL_ENHANCED)) {
                                rc = tipc_msg_fragment(skb, &tnlhdr, tnl->mtu,
                                                       &frags);
                                if (rc) {
                                        pr_warn("%sunable to frag msg: rc %d\n",
                                                link_co_err, rc);
                                        return;
                                }
                                pktcnt += skb_queue_len(&frags) - 1;
                                pktcnt_need_update = true;
                                skb_queue_splice_tail_init(&frags, &tnlq);
                                continue;
                        }
                        /* Unluckily, peer doesn't have TIPC_TUNNEL_ENHANCED
                         * => Just warn it and return!
                         */
                        pr_warn_ratelimited("%stoo large msg <%d, %d>: %d!\n",
                                            link_co_err, msg_user(hdr),
                                            msg_type(hdr), msg_size(hdr));
                        return;
                }

                msg_set_size(&tnlhdr, pktlen + INT_H_SIZE);
                tnlskb = tipc_buf_acquire(pktlen + INT_H_SIZE, GFP_ATOMIC);
                if (!tnlskb) {
                        pr_warn("%sunable to send packet\n", link_co_err);
                        return;
                }
                skb_copy_to_linear_data(tnlskb, &tnlhdr, INT_H_SIZE);
                skb_copy_to_linear_data_offset(tnlskb, INT_H_SIZE, hdr, pktlen);
                __skb_queue_tail(&tnlq, tnlskb);
        }
        if (queue != &l->backlogq) {
                queue = &l->backlogq;
                goto tnl;
        }

        if (pktcnt_need_update)
                skb_queue_walk(&tnlq, skb) {
                        hdr = buf_msg(skb);
                        msg_set_msgcnt(hdr, pktcnt);
                }

        tipc_link_xmit(tnl, &tnlq, xmitq);

        if (mtyp == FAILOVER_MSG) {
                struct sk_buff_head *fdefq = &tnl->failover_deferdq;

                tnl->drop_point = l->rcv_nxt;
                tnl->failover_reasm_skb = l->reasm_buf;
                l->reasm_buf = NULL;

                /* Failover the link's deferdq */
                if (unlikely(!skb_queue_empty(fdefq))) {
                        pr_warn("Link failover deferdq not empty: %d!\n",
                                skb_queue_len(fdefq));
                        __skb_queue_purge(fdefq);
                }
                skb_queue_splice_init(&l->deferdq, fdefq);
        }
}

/**
 * tipc_link_failover_prepare() - prepare tnl for link failover
 *
 * This is a special version of the precursor - tipc_link_tnl_prepare(),
 * see the tipc_node_link_failover() for details
 *
 * @l: failover link
 * @tnl: tunnel link
 * @xmitq: queue for messages to be xmited
 */
void tipc_link_failover_prepare(struct tipc_link *l, struct tipc_link *tnl,
                                struct sk_buff_head *xmitq)
{
        struct sk_buff_head *fdefq = &tnl->failover_deferdq;

        tipc_link_create_dummy_tnl_msg(tnl, xmitq);

        /* This failover link endpoint was never established before,
         * so it has not received anything from peer.
         * Otherwise, it must be a normal failover situation or the
         * node has entered SELF_DOWN_PEER_LEAVING and both peer nodes
         * would have to start over from scratch instead.
         */
        tnl->drop_point = 1;
        tnl->failover_reasm_skb = NULL;

        /* Initiate the link's failover deferdq */
        if (unlikely(!skb_queue_empty(fdefq))) {
                pr_warn("Link failover deferdq not empty: %d!\n",
                        skb_queue_len(fdefq));
                __skb_queue_purge(fdefq);
        }
}

/* tipc_link_validate_msg(): validate message against current link state
 * Returns true if message should be accepted, otherwise false
 */
bool tipc_link_validate_msg(struct tipc_link *l, struct tipc_msg *hdr)
{
        u16 curr_session = l->peer_session;
        u16 session = msg_session(hdr);
        int mtyp = msg_type(hdr);

        if (msg_user(hdr) != LINK_PROTOCOL)
                return true;

        switch (mtyp) {
        case RESET_MSG:
                if (!l->in_session)
                        return true;
                /* Accept only RESET with new session number */
                return more(session, curr_session);
        case ACTIVATE_MSG:
                if (!l->in_session)
                        return true;
                /* Accept only ACTIVATE with new or current session number */
                return !less(session, curr_session);
        case STATE_MSG:
                /* Accept only STATE with current session number */
                if (!l->in_session)
                        return false;
                if (session != curr_session)
                        return false;
                /* Extra sanity check */
                if (!tipc_link_is_up(l) && msg_ack(hdr))
                        return false;
                if (!(l->peer_caps & TIPC_LINK_PROTO_SEQNO))
                        return true;
                /* Accept only STATE with new sequence number */
                return !less(msg_seqno(hdr), l->rcv_nxt_state);
        default:
                return false;
        }
}

/* tipc_link_proto_rcv(): receive link level protocol message :
 * Note that network plane id propagates through the network, and may
 * change at any time. The node with lowest numerical id determines
 * network plane
 */
static int tipc_link_proto_rcv(struct tipc_link *l, struct sk_buff *skb,
                               struct sk_buff_head *xmitq)
{
        struct tipc_msg *hdr = buf_msg(skb);
        struct tipc_gap_ack_blks *ga = NULL;
        bool reply = msg_probe(hdr), retransmitted = false;
        u32 dlen = msg_data_sz(hdr), glen = 0, msg_max;
        u16 peers_snd_nxt =  msg_next_sent(hdr);
        u16 peers_tol = msg_link_tolerance(hdr);
        u16 peers_prio = msg_linkprio(hdr);
        u16 gap = msg_seq_gap(hdr);
        u16 ack = msg_ack(hdr);
        u16 rcv_nxt = l->rcv_nxt;
        u16 rcvgap = 0;
        int mtyp = msg_type(hdr);
        int rc = 0, released;
        char *if_name;
        void *data;

        trace_tipc_proto_rcv(skb, false, l->name);

        if (dlen > U16_MAX)
                goto exit;

        if (tipc_link_is_blocked(l) || !xmitq)
                goto exit;

        if (tipc_own_addr(l->net) > msg_prevnode(hdr))
                l->net_plane = msg_net_plane(hdr);

        if (skb_linearize(skb))
                goto exit;

        hdr = buf_msg(skb);
        data = msg_data(hdr);

        if (!tipc_link_validate_msg(l, hdr)) {
                trace_tipc_skb_dump(skb, false, "PROTO invalid (1)!");
                trace_tipc_link_dump(l, TIPC_DUMP_NONE, "PROTO invalid (1)!");
                goto exit;
        }

        switch (mtyp) {
        case RESET_MSG:
        case ACTIVATE_MSG:
                msg_max = msg_max_pkt(hdr);
                if (msg_max < tipc_bearer_min_mtu(l->net, l->bearer_id))
                        break;
                /* Complete own link name with peer's interface name */
                if_name =  strrchr(l->name, ':') + 1;
                if (sizeof(l->name) - (if_name - l->name) <= TIPC_MAX_IF_NAME)
                        break;
                if (msg_data_sz(hdr) < TIPC_MAX_IF_NAME)
                        break;
                strscpy(if_name, data, TIPC_MAX_IF_NAME);

                /* Update own tolerance if peer indicates a non-zero value */
                if (tipc_in_range(peers_tol, TIPC_MIN_LINK_TOL, TIPC_MAX_LINK_TOL)) {
                        l->tolerance = peers_tol;
                        l->bc_rcvlink->tolerance = peers_tol;
                }
                /* Update own priority if peer's priority is higher */
                if (tipc_in_range(peers_prio, l->priority + 1, TIPC_MAX_LINK_PRI))
                        l->priority = peers_prio;

                /* If peer is going down we want full re-establish cycle */
                if (msg_peer_stopping(hdr)) {
                        rc = tipc_link_fsm_evt(l, LINK_FAILURE_EVT);
                        break;
                }

                /* If this endpoint was re-created while peer was ESTABLISHING
                 * it doesn't know current session number. Force re-synch.
                 */
                if (mtyp == ACTIVATE_MSG && msg_dest_session_valid(hdr) &&
                    l->session != msg_dest_session(hdr)) {
                        if (less(l->session, msg_dest_session(hdr)))
                                l->session = msg_dest_session(hdr) + 1;
                        break;
                }

                /* ACTIVATE_MSG serves as PEER_RESET if link is already down */
                if (mtyp == RESET_MSG || !tipc_link_is_up(l))
                        rc = tipc_link_fsm_evt(l, LINK_PEER_RESET_EVT);

                /* ACTIVATE_MSG takes up link if it was already locally reset */
                if (mtyp == ACTIVATE_MSG && l->state == LINK_ESTABLISHING)
                        rc = TIPC_LINK_UP_EVT;

                l->peer_session = msg_session(hdr);
                l->in_session = true;
                l->peer_bearer_id = msg_bearer_id(hdr);
                if (l->mtu > msg_max)
                        l->mtu = msg_max;
                break;

        case STATE_MSG:
                /* Validate Gap ACK blocks, drop if invalid */
                glen = tipc_get_gap_ack_blks(&ga, l, hdr, true);
                if (glen > dlen)
                        break;

                l->rcv_nxt_state = msg_seqno(hdr) + 1;

                /* Update own tolerance if peer indicates a non-zero value */
                if (tipc_in_range(peers_tol, TIPC_MIN_LINK_TOL, TIPC_MAX_LINK_TOL)) {
                        l->tolerance = peers_tol;
                        l->bc_rcvlink->tolerance = peers_tol;
                }
                /* Update own prio if peer indicates a different value */
                if ((peers_prio != l->priority) &&
                    tipc_in_range(peers_prio, 1, TIPC_MAX_LINK_PRI)) {
                        l->priority = peers_prio;
                        rc = tipc_link_fsm_evt(l, LINK_FAILURE_EVT);
                }

                l->silent_intv_cnt = 0;
                l->stats.recv_states++;
                if (msg_probe(hdr))
                        l->stats.recv_probes++;

                if (!tipc_link_is_up(l)) {
                        if (l->state == LINK_ESTABLISHING)
                                rc = TIPC_LINK_UP_EVT;
                        break;
                }

                tipc_mon_rcv(l->net, data + glen, dlen - glen, l->addr,
                             &l->mon_state, l->bearer_id);

                /* Send NACK if peer has sent pkts we haven't received yet */
                if ((reply || msg_is_keepalive(hdr)) &&
                    more(peers_snd_nxt, rcv_nxt) &&
                    !tipc_link_is_synching(l) &&
                    skb_queue_empty(&l->deferdq))
                        rcvgap = peers_snd_nxt - l->rcv_nxt;
                if (rcvgap || reply)
                        tipc_link_build_proto_msg(l, STATE_MSG, 0, reply,
                                                  rcvgap, 0, 0, xmitq);

                released = tipc_link_advance_transmq(l, l, ack, gap, ga, xmitq,
                                                     &retransmitted, &rc);
                if (gap)
                        l->stats.recv_nacks++;
                if (released || retransmitted)
                        tipc_link_update_cwin(l, released, retransmitted);
                if (released)
                        tipc_link_advance_backlog(l, xmitq);
                if (unlikely(!skb_queue_empty(&l->wakeupq)))
                        link_prepare_wakeup(l);
        }
exit:
        kfree_skb(skb);
        return rc;
}

/* tipc_link_build_bc_proto_msg() - create broadcast protocol message
 */
static bool tipc_link_build_bc_proto_msg(struct tipc_link *l, bool bcast,
                                         u16 peers_snd_nxt,
                                         struct sk_buff_head *xmitq)
{
        struct sk_buff *skb;
        struct tipc_msg *hdr;
        struct sk_buff *dfrd_skb = skb_peek(&l->deferdq);
        u16 ack = l->rcv_nxt - 1;
        u16 gap_to = peers_snd_nxt - 1;

        skb = tipc_msg_create(BCAST_PROTOCOL, STATE_MSG, INT_H_SIZE,
                              0, l->addr, tipc_own_addr(l->net), 0, 0, 0);
        if (!skb)
                return false;
        hdr = buf_msg(skb);
        msg_set_last_bcast(hdr, l->bc_sndlink->snd_nxt - 1);
        msg_set_bcast_ack(hdr, ack);
        msg_set_bcgap_after(hdr, ack);
        if (dfrd_skb)
                gap_to = buf_seqno(dfrd_skb) - 1;
        msg_set_bcgap_to(hdr, gap_to);
        msg_set_non_seq(hdr, bcast);
        __skb_queue_tail(xmitq, skb);
        return true;
}

/* tipc_link_build_bc_init_msg() - synchronize broadcast link endpoints.
 *
 * Give a newly added peer node the sequence number where it should
 * start receiving and acking broadcast packets.
 */
static void tipc_link_build_bc_init_msg(struct tipc_link *l,
                                        struct sk_buff_head *xmitq)
{
        struct sk_buff_head list;

        __skb_queue_head_init(&list);
        if (!tipc_link_build_bc_proto_msg(l->bc_rcvlink, false, 0, &list))
                return;
        msg_set_bc_ack_invalid(buf_msg(skb_peek(&list)), true);
        tipc_link_xmit(l, &list, xmitq);
}

/* tipc_link_bc_init_rcv - receive initial broadcast synch data from peer
 */
void tipc_link_bc_init_rcv(struct tipc_link *l, struct tipc_msg *hdr)
{
        int mtyp = msg_type(hdr);
        u16 peers_snd_nxt = msg_bc_snd_nxt(hdr);

        if (tipc_link_is_up(l))
                return;

        if (msg_user(hdr) == BCAST_PROTOCOL) {
                l->rcv_nxt = peers_snd_nxt;
                l->state = LINK_ESTABLISHED;
                return;
        }

        if (l->peer_caps & TIPC_BCAST_SYNCH)
                return;

        if (msg_peer_node_is_up(hdr))
                return;

        /* Compatibility: accept older, less safe initial synch data */
        if ((mtyp == RESET_MSG) || (mtyp == ACTIVATE_MSG))
                l->rcv_nxt = peers_snd_nxt;
}

/* tipc_link_bc_sync_rcv - update rcv link according to peer's send state
 */
int tipc_link_bc_sync_rcv(struct tipc_link *l, struct tipc_msg *hdr,
                          struct sk_buff_head *xmitq)
{
        u16 peers_snd_nxt = msg_bc_snd_nxt(hdr);
        int rc = 0;

        if (!tipc_link_is_up(l))
                return rc;

        if (!msg_peer_node_is_up(hdr))
                return rc;

        /* Open when peer acknowledges our bcast init msg (pkt #1) */
        if (msg_ack(hdr))
                l->bc_peer_is_up = true;

        if (!l->bc_peer_is_up)
                return rc;

        /* Ignore if peers_snd_nxt goes beyond receive window */
        if (more(peers_snd_nxt, l->rcv_nxt + l->window))
                return rc;

        l->snd_nxt = peers_snd_nxt;
        if (link_bc_rcv_gap(l))
                rc |= TIPC_LINK_SND_STATE;

        /* Return now if sender supports nack via STATE messages */
        if (l->peer_caps & TIPC_BCAST_STATE_NACK)
                return rc;

        /* Otherwise, be backwards compatible */

        if (!more(peers_snd_nxt, l->rcv_nxt)) {
                l->nack_state = BC_NACK_SND_CONDITIONAL;
                return 0;
        }

        /* Don't NACK if one was recently sent or peeked */
        if (l->nack_state == BC_NACK_SND_SUPPRESS) {
                l->nack_state = BC_NACK_SND_UNCONDITIONAL;
                return 0;
        }

        /* Conditionally delay NACK sending until next synch rcv */
        if (l->nack_state == BC_NACK_SND_CONDITIONAL) {
                l->nack_state = BC_NACK_SND_UNCONDITIONAL;
                if ((peers_snd_nxt - l->rcv_nxt) < TIPC_MIN_LINK_WIN)
                        return 0;
        }

        /* Send NACK now but suppress next one */
        tipc_link_build_bc_proto_msg(l, true, peers_snd_nxt, xmitq);
        l->nack_state = BC_NACK_SND_SUPPRESS;
        return 0;
}

int tipc_link_bc_ack_rcv(struct tipc_link *r, u16 acked, u16 gap,
                         struct tipc_gap_ack_blks *ga,
                         struct sk_buff_head *xmitq,
                         struct sk_buff_head *retrq)
{
        struct tipc_link *l = r->bc_sndlink;
        bool unused = false;
        int rc = 0;

        if (!tipc_link_is_up(r) || !r->bc_peer_is_up)
                return 0;

        if (gap) {
                l->stats.recv_nacks++;
                r->stats.recv_nacks++;
        }

        if (less(acked, r->acked) || (acked == r->acked && !gap && !ga))
                return 0;

        trace_tipc_link_bc_ack(r, acked, gap, &l->transmq);
        tipc_link_advance_transmq(l, r, acked, gap, ga, retrq, &unused, &rc);

        tipc_link_advance_backlog(l, xmitq);
        if (unlikely(!skb_queue_empty(&l->wakeupq)))
                link_prepare_wakeup(l);

        return rc;
}

/* tipc_link_bc_nack_rcv(): receive broadcast nack message
 * This function is here for backwards compatibility, since
 * no BCAST_PROTOCOL/STATE messages occur from TIPC v2.5.
 */
int tipc_link_bc_nack_rcv(struct tipc_link *l, struct sk_buff *skb,
                          struct sk_buff_head *xmitq)
{
        struct tipc_msg *hdr = buf_msg(skb);
        u32 dnode = msg_destnode(hdr);
        int mtyp = msg_type(hdr);
        u16 acked = msg_bcast_ack(hdr);
        u16 from = acked + 1;
        u16 to = msg_bcgap_to(hdr);
        u16 peers_snd_nxt = to + 1;
        int rc = 0;

        kfree_skb(skb);

        if (!tipc_link_is_up(l) || !l->bc_peer_is_up)
                return 0;

        if (mtyp != STATE_MSG)
                return 0;

        if (dnode == tipc_own_addr(l->net)) {
                rc = tipc_link_bc_ack_rcv(l, acked, to - acked, NULL, xmitq,
                                          xmitq);
                l->stats.recv_nacks++;
                return rc;
        }

        /* Msg for other node => suppress own NACK at next sync if applicable */
        if (more(peers_snd_nxt, l->rcv_nxt) && !less(l->rcv_nxt, from))
                l->nack_state = BC_NACK_SND_SUPPRESS;

        return 0;
}

void tipc_link_set_queue_limits(struct tipc_link *l, u32 min_win, u32 max_win)
{
        int max_bulk = TIPC_MAX_PUBL / (l->mtu / ITEM_SIZE);

        l->min_win = min_win;
        l->ssthresh = max_win;
        l->max_win = max_win;
        l->window = min_win;
        l->backlog[TIPC_LOW_IMPORTANCE].limit      = min_win * 2;
        l->backlog[TIPC_MEDIUM_IMPORTANCE].limit   = min_win * 4;
        l->backlog[TIPC_HIGH_IMPORTANCE].limit     = min_win * 6;
        l->backlog[TIPC_CRITICAL_IMPORTANCE].limit = min_win * 8;
        l->backlog[TIPC_SYSTEM_IMPORTANCE].limit   = max_bulk;
}

/**
 * tipc_link_reset_stats - reset link statistics
 * @l: pointer to link
 */
void tipc_link_reset_stats(struct tipc_link *l)
{
        memset(&l->stats, 0, sizeof(l->stats));
}

static void link_print(struct tipc_link *l, const char *str)
{
        struct sk_buff *hskb = skb_peek(&l->transmq);
        u16 head = hskb ? msg_seqno(buf_msg(hskb)) : l->snd_nxt - 1;
        u16 tail = l->snd_nxt - 1;

        pr_info("%s Link <%s> state %x\n", str, l->name, l->state);
        pr_info("XMTQ: %u [%u-%u], BKLGQ: %u, SNDNX: %u, RCVNX: %u\n",
                skb_queue_len(&l->transmq), head, tail,
                skb_queue_len(&l->backlogq), l->snd_nxt, l->rcv_nxt);
}

/* Parse and validate nested (link) properties valid for media, bearer and link
 */
int tipc_nl_parse_link_prop(struct nlattr *prop, struct nlattr *props[])
{
        int err;

        err = nla_parse_nested_deprecated(props, TIPC_NLA_PROP_MAX, prop,
                                          tipc_nl_prop_policy, NULL);
        if (err)
                return err;

        if (props[TIPC_NLA_PROP_PRIO]) {
                u32 prio;

                prio = nla_get_u32(props[TIPC_NLA_PROP_PRIO]);
                if (prio > TIPC_MAX_LINK_PRI)
                        return -EINVAL;
        }

        if (props[TIPC_NLA_PROP_TOL]) {
                u32 tol;

                tol = nla_get_u32(props[TIPC_NLA_PROP_TOL]);
                if ((tol < TIPC_MIN_LINK_TOL) || (tol > TIPC_MAX_LINK_TOL))
                        return -EINVAL;
        }

        if (props[TIPC_NLA_PROP_WIN]) {
                u32 max_win;

                max_win = nla_get_u32(props[TIPC_NLA_PROP_WIN]);
                if (max_win < TIPC_DEF_LINK_WIN || max_win > TIPC_MAX_LINK_WIN)
                        return -EINVAL;
        }

        return 0;
}

static int __tipc_nl_add_stats(struct sk_buff *skb, struct tipc_stats *s)
{
        int i;
        struct nlattr *stats;

        struct nla_map {
                u32 key;
                u32 val;
        };

        struct nla_map map[] = {
                {TIPC_NLA_STATS_RX_INFO, 0},
                {TIPC_NLA_STATS_RX_FRAGMENTS, s->recv_fragments},
                {TIPC_NLA_STATS_RX_FRAGMENTED, s->recv_fragmented},
                {TIPC_NLA_STATS_RX_BUNDLES, s->recv_bundles},
                {TIPC_NLA_STATS_RX_BUNDLED, s->recv_bundled},
                {TIPC_NLA_STATS_TX_INFO, 0},
                {TIPC_NLA_STATS_TX_FRAGMENTS, s->sent_fragments},
                {TIPC_NLA_STATS_TX_FRAGMENTED, s->sent_fragmented},
                {TIPC_NLA_STATS_TX_BUNDLES, s->sent_bundles},
                {TIPC_NLA_STATS_TX_BUNDLED, s->sent_bundled},
                {TIPC_NLA_STATS_MSG_PROF_TOT, (s->msg_length_counts) ?
                        s->msg_length_counts : 1},
                {TIPC_NLA_STATS_MSG_LEN_CNT, s->msg_length_counts},
                {TIPC_NLA_STATS_MSG_LEN_TOT, s->msg_lengths_total},
                {TIPC_NLA_STATS_MSG_LEN_P0, s->msg_length_profile[0]},
                {TIPC_NLA_STATS_MSG_LEN_P1, s->msg_length_profile[1]},
                {TIPC_NLA_STATS_MSG_LEN_P2, s->msg_length_profile[2]},
                {TIPC_NLA_STATS_MSG_LEN_P3, s->msg_length_profile[3]},
                {TIPC_NLA_STATS_MSG_LEN_P4, s->msg_length_profile[4]},
                {TIPC_NLA_STATS_MSG_LEN_P5, s->msg_length_profile[5]},
                {TIPC_NLA_STATS_MSG_LEN_P6, s->msg_length_profile[6]},
                {TIPC_NLA_STATS_RX_STATES, s->recv_states},
                {TIPC_NLA_STATS_RX_PROBES, s->recv_probes},
                {TIPC_NLA_STATS_RX_NACKS, s->recv_nacks},
                {TIPC_NLA_STATS_RX_DEFERRED, s->deferred_recv},
                {TIPC_NLA_STATS_TX_STATES, s->sent_states},
                {TIPC_NLA_STATS_TX_PROBES, s->sent_probes},
                {TIPC_NLA_STATS_TX_NACKS, s->sent_nacks},
                {TIPC_NLA_STATS_TX_ACKS, s->sent_acks},
                {TIPC_NLA_STATS_RETRANSMITTED, s->retransmitted},
                {TIPC_NLA_STATS_DUPLICATES, s->duplicates},
                {TIPC_NLA_STATS_LINK_CONGS, s->link_congs},
                {TIPC_NLA_STATS_MAX_QUEUE, s->max_queue_sz},
                {TIPC_NLA_STATS_AVG_QUEUE, s->queue_sz_counts ?
                        (s->accu_queue_sz / s->queue_sz_counts) : 0}
        };

        stats = nla_nest_start_noflag(skb, TIPC_NLA_LINK_STATS);
        if (!stats)
                return -EMSGSIZE;

        for (i = 0; i <  ARRAY_SIZE(map); i++)
                if (nla_put_u32(skb, map[i].key, map[i].val))
                        goto msg_full;

        nla_nest_end(skb, stats);

        return 0;
msg_full:
        nla_nest_cancel(skb, stats);

        return -EMSGSIZE;
}

/* Caller should hold appropriate locks to protect the link */
int __tipc_nl_add_link(struct net *net, struct tipc_nl_msg *msg,
                       struct tipc_link *link, int nlflags)
{
        u32 self = tipc_own_addr(net);
        struct nlattr *attrs;
        struct nlattr *prop;
        void *hdr;
        int err;

        hdr = genlmsg_put(msg->skb, msg->portid, msg->seq, &tipc_genl_family,
                          nlflags, TIPC_NL_LINK_GET);
        if (!hdr)
                return -EMSGSIZE;

        attrs = nla_nest_start_noflag(msg->skb, TIPC_NLA_LINK);
        if (!attrs)
                goto msg_full;

        if (nla_put_string(msg->skb, TIPC_NLA_LINK_NAME, link->name))
                goto attr_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_LINK_DEST, tipc_cluster_mask(self)))
                goto attr_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_LINK_MTU, link->mtu))
                goto attr_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_LINK_RX, link->stats.recv_pkts))
                goto attr_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_LINK_TX, link->stats.sent_pkts))
                goto attr_msg_full;

        if (tipc_link_is_up(link))
                if (nla_put_flag(msg->skb, TIPC_NLA_LINK_UP))
                        goto attr_msg_full;
        if (link->active)
                if (nla_put_flag(msg->skb, TIPC_NLA_LINK_ACTIVE))
                        goto attr_msg_full;

        prop = nla_nest_start_noflag(msg->skb, TIPC_NLA_LINK_PROP);
        if (!prop)
                goto attr_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_PROP_PRIO, link->priority))
                goto prop_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_PROP_TOL, link->tolerance))
                goto prop_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_PROP_WIN,
                        link->window))
                goto prop_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_PROP_PRIO, link->priority))
                goto prop_msg_full;
        nla_nest_end(msg->skb, prop);

        err = __tipc_nl_add_stats(msg->skb, &link->stats);
        if (err)
                goto attr_msg_full;

        nla_nest_end(msg->skb, attrs);
        genlmsg_end(msg->skb, hdr);

        return 0;

prop_msg_full:
        nla_nest_cancel(msg->skb, prop);
attr_msg_full:
        nla_nest_cancel(msg->skb, attrs);
msg_full:
        genlmsg_cancel(msg->skb, hdr);

        return -EMSGSIZE;
}

static int __tipc_nl_add_bc_link_stat(struct sk_buff *skb,
                                      struct tipc_stats *stats)
{
        int i;
        struct nlattr *nest;

        struct nla_map {
                __u32 key;
                __u32 val;
        };

        struct nla_map map[] = {
                {TIPC_NLA_STATS_RX_INFO, stats->recv_pkts},
                {TIPC_NLA_STATS_RX_FRAGMENTS, stats->recv_fragments},
                {TIPC_NLA_STATS_RX_FRAGMENTED, stats->recv_fragmented},
                {TIPC_NLA_STATS_RX_BUNDLES, stats->recv_bundles},
                {TIPC_NLA_STATS_RX_BUNDLED, stats->recv_bundled},
                {TIPC_NLA_STATS_TX_INFO, stats->sent_pkts},
                {TIPC_NLA_STATS_TX_FRAGMENTS, stats->sent_fragments},
                {TIPC_NLA_STATS_TX_FRAGMENTED, stats->sent_fragmented},
                {TIPC_NLA_STATS_TX_BUNDLES, stats->sent_bundles},
                {TIPC_NLA_STATS_TX_BUNDLED, stats->sent_bundled},
                {TIPC_NLA_STATS_RX_NACKS, stats->recv_nacks},
                {TIPC_NLA_STATS_RX_DEFERRED, stats->deferred_recv},
                {TIPC_NLA_STATS_TX_NACKS, stats->sent_nacks},
                {TIPC_NLA_STATS_TX_ACKS, stats->sent_acks},
                {TIPC_NLA_STATS_RETRANSMITTED, stats->retransmitted},
                {TIPC_NLA_STATS_DUPLICATES, stats->duplicates},
                {TIPC_NLA_STATS_LINK_CONGS, stats->link_congs},
                {TIPC_NLA_STATS_MAX_QUEUE, stats->max_queue_sz},
                {TIPC_NLA_STATS_AVG_QUEUE, stats->queue_sz_counts ?
                        (stats->accu_queue_sz / stats->queue_sz_counts) : 0}
        };

        nest = nla_nest_start_noflag(skb, TIPC_NLA_LINK_STATS);
        if (!nest)
                return -EMSGSIZE;

        for (i = 0; i <  ARRAY_SIZE(map); i++)
                if (nla_put_u32(skb, map[i].key, map[i].val))
                        goto msg_full;

        nla_nest_end(skb, nest);

        return 0;
msg_full:
        nla_nest_cancel(skb, nest);

        return -EMSGSIZE;
}

int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg,
                        struct tipc_link *bcl)
{
        int err;
        void *hdr;
        struct nlattr *attrs;
        struct nlattr *prop;
        u32 bc_mode = tipc_bcast_get_mode(net);
        u32 bc_ratio = tipc_bcast_get_broadcast_ratio(net);

        if (!bcl)
                return 0;

        tipc_bcast_lock(net);

        hdr = genlmsg_put(msg->skb, msg->portid, msg->seq, &tipc_genl_family,
                          NLM_F_MULTI, TIPC_NL_LINK_GET);
        if (!hdr) {
                tipc_bcast_unlock(net);
                return -EMSGSIZE;
        }

        attrs = nla_nest_start_noflag(msg->skb, TIPC_NLA_LINK);
        if (!attrs)
                goto msg_full;

        /* The broadcast link is always up */
        if (nla_put_flag(msg->skb, TIPC_NLA_LINK_UP))
                goto attr_msg_full;

        if (nla_put_flag(msg->skb, TIPC_NLA_LINK_BROADCAST))
                goto attr_msg_full;
        if (nla_put_string(msg->skb, TIPC_NLA_LINK_NAME, bcl->name))
                goto attr_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_LINK_RX, 0))
                goto attr_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_LINK_TX, 0))
                goto attr_msg_full;

        prop = nla_nest_start_noflag(msg->skb, TIPC_NLA_LINK_PROP);
        if (!prop)
                goto attr_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_PROP_WIN, bcl->max_win))
                goto prop_msg_full;
        if (nla_put_u32(msg->skb, TIPC_NLA_PROP_BROADCAST, bc_mode))
                goto prop_msg_full;
        if (bc_mode & BCLINK_MODE_SEL)
                if (nla_put_u32(msg->skb, TIPC_NLA_PROP_BROADCAST_RATIO,
                                bc_ratio))
                        goto prop_msg_full;
        nla_nest_end(msg->skb, prop);

        err = __tipc_nl_add_bc_link_stat(msg->skb, &bcl->stats);
        if (err)
                goto attr_msg_full;

        tipc_bcast_unlock(net);
        nla_nest_end(msg->skb, attrs);
        genlmsg_end(msg->skb, hdr);

        return 0;

prop_msg_full:
        nla_nest_cancel(msg->skb, prop);
attr_msg_full:
        nla_nest_cancel(msg->skb, attrs);
msg_full:
        tipc_bcast_unlock(net);
        genlmsg_cancel(msg->skb, hdr);

        return -EMSGSIZE;
}

void tipc_link_set_tolerance(struct tipc_link *l, u32 tol,
                             struct sk_buff_head *xmitq)
{
        l->tolerance = tol;
        if (l->bc_rcvlink)
                l->bc_rcvlink->tolerance = tol;
        if (tipc_link_is_up(l))
                tipc_link_build_proto_msg(l, STATE_MSG, 0, 0, 0, tol, 0, xmitq);
}

void tipc_link_set_prio(struct tipc_link *l, u32 prio,
                        struct sk_buff_head *xmitq)
{
        l->priority = prio;
        tipc_link_build_proto_msg(l, STATE_MSG, 0, 0, 0, 0, prio, xmitq);
}

void tipc_link_set_abort_limit(struct tipc_link *l, u32 limit)
{
        l->abort_limit = limit;
}

/**
 * tipc_link_dump - dump TIPC link data
 * @l: tipc link to be dumped
 * @dqueues: bitmask to decide if any link queue to be dumped?
 *           - TIPC_DUMP_NONE: don't dump link queues
 *           - TIPC_DUMP_TRANSMQ: dump link transmq queue
 *           - TIPC_DUMP_BACKLOGQ: dump link backlog queue
 *           - TIPC_DUMP_DEFERDQ: dump link deferd queue
 *           - TIPC_DUMP_INPUTQ: dump link input queue
 *           - TIPC_DUMP_WAKEUP: dump link wakeup queue
 *           - TIPC_DUMP_ALL: dump all the link queues above
 * @buf: returned buffer of dump data in format
 */
int tipc_link_dump(struct tipc_link *l, u16 dqueues, char *buf)
{
        int i = 0;
        size_t sz = (dqueues) ? LINK_LMAX : LINK_LMIN;
        struct sk_buff_head *list;
        struct sk_buff *hskb, *tskb;
        u32 len;

        if (!l) {
                i += scnprintf(buf, sz, "link data: (null)\n");
                return i;
        }

        i += scnprintf(buf, sz, "link data: %x", l->addr);
        i += scnprintf(buf + i, sz - i, " %x", l->state);
        i += scnprintf(buf + i, sz - i, " %u", l->in_session);
        i += scnprintf(buf + i, sz - i, " %u", l->session);
        i += scnprintf(buf + i, sz - i, " %u", l->peer_session);
        i += scnprintf(buf + i, sz - i, " %u", l->snd_nxt);
        i += scnprintf(buf + i, sz - i, " %u", l->rcv_nxt);
        i += scnprintf(buf + i, sz - i, " %u", l->snd_nxt_state);
        i += scnprintf(buf + i, sz - i, " %u", l->rcv_nxt_state);
        i += scnprintf(buf + i, sz - i, " %x", l->peer_caps);
        i += scnprintf(buf + i, sz - i, " %u", l->silent_intv_cnt);
        i += scnprintf(buf + i, sz - i, " %u", l->rst_cnt);
        i += scnprintf(buf + i, sz - i, " %u", 0);
        i += scnprintf(buf + i, sz - i, " %u", 0);
        i += scnprintf(buf + i, sz - i, " %u", l->acked);

        list = &l->transmq;
        len = skb_queue_len(list);
        hskb = skb_peek(list);
        tskb = skb_peek_tail(list);
        i += scnprintf(buf + i, sz - i, " | %u %u %u", len,
                       (hskb) ? msg_seqno(buf_msg(hskb)) : 0,
                       (tskb) ? msg_seqno(buf_msg(tskb)) : 0);

        list = &l->deferdq;
        len = skb_queue_len(list);
        hskb = skb_peek(list);
        tskb = skb_peek_tail(list);
        i += scnprintf(buf + i, sz - i, " | %u %u %u", len,
                       (hskb) ? msg_seqno(buf_msg(hskb)) : 0,
                       (tskb) ? msg_seqno(buf_msg(tskb)) : 0);

        list = &l->backlogq;
        len = skb_queue_len(list);
        hskb = skb_peek(list);
        tskb = skb_peek_tail(list);
        i += scnprintf(buf + i, sz - i, " | %u %u %u", len,
                       (hskb) ? msg_seqno(buf_msg(hskb)) : 0,
                       (tskb) ? msg_seqno(buf_msg(tskb)) : 0);

        list = l->inputq;
        len = skb_queue_len(list);
        hskb = skb_peek(list);
        tskb = skb_peek_tail(list);
        i += scnprintf(buf + i, sz - i, " | %u %u %u\n", len,
                       (hskb) ? msg_seqno(buf_msg(hskb)) : 0,
                       (tskb) ? msg_seqno(buf_msg(tskb)) : 0);

        if (dqueues & TIPC_DUMP_TRANSMQ) {
                i += scnprintf(buf + i, sz - i, "transmq: ");
                i += tipc_list_dump(&l->transmq, false, buf + i);
        }
        if (dqueues & TIPC_DUMP_BACKLOGQ) {
                i += scnprintf(buf + i, sz - i,
                               "backlogq: <%u %u %u %u %u>, ",
                               l->backlog[TIPC_LOW_IMPORTANCE].len,
                               l->backlog[TIPC_MEDIUM_IMPORTANCE].len,
                               l->backlog[TIPC_HIGH_IMPORTANCE].len,
                               l->backlog[TIPC_CRITICAL_IMPORTANCE].len,
                               l->backlog[TIPC_SYSTEM_IMPORTANCE].len);
                i += tipc_list_dump(&l->backlogq, false, buf + i);
        }
        if (dqueues & TIPC_DUMP_DEFERDQ) {
                i += scnprintf(buf + i, sz - i, "deferdq: ");
                i += tipc_list_dump(&l->deferdq, false, buf + i);
        }
        if (dqueues & TIPC_DUMP_INPUTQ) {
                i += scnprintf(buf + i, sz - i, "inputq: ");
                i += tipc_list_dump(l->inputq, false, buf + i);
        }
        if (dqueues & TIPC_DUMP_WAKEUP) {
                i += scnprintf(buf + i, sz - i, "wakeup: ");
                i += tipc_list_dump(&l->wakeupq, false, buf + i);
        }

        return i;
}
















































































































 1354 



 1355 















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   11 












   11 
   11 










   11 
























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
/*
 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "ipoib.h"

#include <linux/module.h>

#include <linux/init.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/vmalloc.h>

#include <linux/if_arp.h>        /* For ARPHRD_xxx */

#include <linux/ip.h>
#include <linux/in.h>

#include <linux/jhash.h>
#include <net/arp.h>
#include <net/addrconf.h>
#include <net/netdev_lock.h>
#include <net/pkt_sched.h>
#include <linux/inetdevice.h>
#include <rdma/ib_cache.h>

MODULE_AUTHOR("Roland Dreier");
MODULE_DESCRIPTION("IP-over-InfiniBand net driver");
MODULE_LICENSE("Dual BSD/GPL");

int ipoib_sendq_size __read_mostly = IPOIB_TX_RING_SIZE;
int ipoib_recvq_size __read_mostly = IPOIB_RX_RING_SIZE;

module_param_named(send_queue_size, ipoib_sendq_size, int, 0444);
MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue");
module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444);
MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue");

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
int ipoib_debug_level;

module_param_named(debug_level, ipoib_debug_level, int, 0644);
MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
#endif

struct ipoib_path_iter {
        struct net_device *dev;
        struct ipoib_path  path;
};

static const u8 ipv4_bcast_addr[] = {
        0x00, 0xff, 0xff, 0xff,
        0xff, 0x12, 0x40, 0x1b,        0x00, 0x00, 0x00, 0x00,
        0x00, 0x00, 0x00, 0x00,        0xff, 0xff, 0xff, 0xff
};

struct workqueue_struct *ipoib_workqueue;

struct ib_sa_client ipoib_sa_client;

static int ipoib_add_one(struct ib_device *device);
static void ipoib_remove_one(struct ib_device *device, void *client_data);
static void ipoib_neigh_reclaim(struct rcu_head *rp);
static struct net_device *ipoib_get_net_dev_by_params(
                struct ib_device *dev, u32 port, u16 pkey,
                const union ib_gid *gid, const struct sockaddr *addr,
                void *client_data);
static int ipoib_set_mac(struct net_device *dev, void *addr);
static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
                       int cmd);

static struct ib_client ipoib_client = {
        .name   = "ipoib",
        .add    = ipoib_add_one,
        .remove = ipoib_remove_one,
        .get_net_dev_by_params = ipoib_get_net_dev_by_params,
};

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
static int ipoib_netdev_event(struct notifier_block *this,
                              unsigned long event, void *ptr)
{
        struct netdev_notifier_info *ni = ptr;
        struct net_device *dev = ni->dev;

        if (dev->netdev_ops->ndo_open != ipoib_open)
                return NOTIFY_DONE;

        switch (event) {
        case NETDEV_REGISTER:
                ipoib_create_debug_files(dev);
                break;
        case NETDEV_CHANGENAME:
                ipoib_delete_debug_files(dev);
                ipoib_create_debug_files(dev);
                break;
        case NETDEV_UNREGISTER:
                ipoib_delete_debug_files(dev);
                break;
        }

        return NOTIFY_DONE;
}
#endif

struct ipoib_ifupdown_work {
        struct work_struct work;
        struct net_device *dev;
        netdevice_tracker dev_tracker;
        bool up;
};

static void ipoib_ifupdown_task(struct work_struct *work)
{
        struct ipoib_ifupdown_work *pwork =
                container_of(work, struct ipoib_ifupdown_work, work);
        struct net_device *dev = pwork->dev;
        unsigned int flags;

        rtnl_lock();
        flags = dev->flags;
        if (pwork->up)
                flags |= IFF_UP;
        else
                flags &= ~IFF_UP;

        if (dev->flags != flags)
                dev_change_flags(dev, flags, NULL);
        rtnl_unlock();
        netdev_put(dev, &pwork->dev_tracker);
        kfree(pwork);
}

static void ipoib_schedule_ifupdown_task(struct net_device *dev, bool up)
{
        struct ipoib_ifupdown_work *work;

        if ((up && (dev->flags & IFF_UP)) ||
            (!up && !(dev->flags & IFF_UP)))
                return;

        work = kmalloc_obj(*work);
        if (!work)
                return;
        work->dev = dev;
        netdev_hold(dev, &work->dev_tracker, GFP_KERNEL);
        work->up = up;
        INIT_WORK(&work->work, ipoib_ifupdown_task);
        queue_work(ipoib_workqueue, &work->work);
}

int ipoib_open(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_dbg(priv, "bringing up interface\n");

        netif_carrier_off(dev);

        set_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);

        if (ipoib_ib_dev_open(dev)) {
                if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
                        return 0;
                goto err_disable;
        }

        ipoib_ib_dev_up(dev);

        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
                struct ipoib_dev_priv *cpriv;

                /* Bring up any child interfaces too */
                netdev_lock_ops_to_full(dev);
                list_for_each_entry(cpriv, &priv->child_intfs, list)
                        ipoib_schedule_ifupdown_task(cpriv->dev, true);
                netdev_unlock_full_to_ops(dev);
        } else if (priv->parent) {
                struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);

                if (!test_bit(IPOIB_FLAG_ADMIN_UP, &ppriv->flags))
                        ipoib_dbg(priv, "parent device %s is not up, so child device may be not functioning.\n",
                                  ppriv->dev->name);
        }
        netif_start_queue(dev);

        return 0;

err_disable:
        clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);

        return -EINVAL;
}

static int ipoib_stop(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_dbg(priv, "stopping interface\n");

        clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);

        netif_stop_queue(dev);

        ipoib_ib_dev_down(dev);
        ipoib_ib_dev_stop(dev);

        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
                struct ipoib_dev_priv *cpriv;

                /* Bring down any child interfaces too */
                netdev_lock_ops_to_full(dev);
                list_for_each_entry(cpriv, &priv->child_intfs, list)
                        ipoib_schedule_ifupdown_task(cpriv->dev, false);
                netdev_unlock_full_to_ops(dev);
        }

        return 0;
}

static netdev_features_t ipoib_fix_features(struct net_device *dev, netdev_features_t features)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags))
                features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO);

        return features;
}

static int ipoib_change_mtu(struct net_device *dev, int new_mtu)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        int ret = 0;

        /* dev->mtu > 2K ==> connected mode */
        if (ipoib_cm_admin_enabled(dev)) {
                if (new_mtu > ipoib_cm_max_mtu(dev))
                        return -EINVAL;

                if (new_mtu > priv->mcast_mtu)
                        ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n",
                                   priv->mcast_mtu);

                WRITE_ONCE(dev->mtu, new_mtu);
                return 0;
        }

        if (new_mtu < (ETH_MIN_MTU + IPOIB_ENCAP_LEN) ||
            new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu))
                return -EINVAL;

        priv->admin_mtu = new_mtu;

        if (priv->mcast_mtu < priv->admin_mtu)
                ipoib_dbg(priv, "MTU must be smaller than the underlying "
                                "link layer MTU - 4 (%u)\n", priv->mcast_mtu);

        new_mtu = min(priv->mcast_mtu, priv->admin_mtu);

        if (priv->rn_ops->ndo_change_mtu) {
                bool carrier_status = netif_carrier_ok(dev);

                netif_carrier_off(dev);

                /* notify lower level on the real mtu */
                ret = priv->rn_ops->ndo_change_mtu(dev, new_mtu);

                if (carrier_status)
                        netif_carrier_on(dev);
        } else {
                WRITE_ONCE(dev->mtu, new_mtu);
        }

        return ret;
}

static void ipoib_get_stats(struct net_device *dev,
                            struct rtnl_link_stats64 *stats)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (priv->rn_ops->ndo_get_stats64)
                priv->rn_ops->ndo_get_stats64(dev, stats);
        else
                netdev_stats_to_stats64(stats, &dev->stats);
}

/* Called with an RCU read lock taken */
static bool ipoib_is_dev_match_addr_rcu(const struct sockaddr *addr,
                                        struct net_device *dev)
{
        struct net *net = dev_net(dev);
        struct in_device *in_dev;
        struct sockaddr_in *addr_in = (struct sockaddr_in *)addr;
        struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr;
        __be32 ret_addr;

        switch (addr->sa_family) {
        case AF_INET:
                in_dev = in_dev_get(dev);
                if (!in_dev)
                        return false;

                ret_addr = inet_confirm_addr(net, in_dev, 0,
                                             addr_in->sin_addr.s_addr,
                                             RT_SCOPE_HOST);
                in_dev_put(in_dev);
                if (ret_addr)
                        return true;

                break;
        case AF_INET6:
                if (IS_ENABLED(CONFIG_IPV6) &&
                    ipv6_chk_addr(net, &addr_in6->sin6_addr, dev, 1))
                        return true;

                break;
        }
        return false;
}

/*
 * Find the L2 master net_device on top of the given net_device.
 * @dev: base IPoIB net_device
 *
 * Returns the L2 master net_device with reference held if the L2 master
 * exists (such as bond netdevice), or returns same netdev with reference
 * held when master does not exist or when L3 master (such as VRF netdev).
 */
static struct net_device *ipoib_get_master_net_dev(struct net_device *dev)
{
        struct net_device *master;

        rcu_read_lock();

        master = netdev_master_upper_dev_get_rcu(dev);
        if (!master || netif_is_l3_master(master))
                master = dev;

        dev_hold(master);
        rcu_read_unlock();

        return master;
}

struct ipoib_walk_data {
        const struct sockaddr *addr;
        struct net_device *result;
};

static int ipoib_upper_walk(struct net_device *upper,
                            struct netdev_nested_priv *priv)
{
        struct ipoib_walk_data *data = (struct ipoib_walk_data *)priv->data;
        int ret = 0;

        if (ipoib_is_dev_match_addr_rcu(data->addr, upper)) {
                dev_hold(upper);
                data->result = upper;
                ret = 1;
        }

        return ret;
}

/**
 * ipoib_get_net_dev_match_addr - Find a net_device matching
 * the given address, which is an upper device of the given net_device.
 *
 * @addr: IP address to look for.
 * @dev: base IPoIB net_device
 *
 * If found, returns the net_device with a reference held. Otherwise return
 * NULL.
 */
static struct net_device *ipoib_get_net_dev_match_addr(
                const struct sockaddr *addr, struct net_device *dev)
{
        struct netdev_nested_priv priv;
        struct ipoib_walk_data data = {
                .addr = addr,
        };

        priv.data = (void *)&data;
        rcu_read_lock();
        if (ipoib_is_dev_match_addr_rcu(addr, dev)) {
                dev_hold(dev);
                data.result = dev;
                goto out;
        }

        netdev_walk_all_upper_dev_rcu(dev, ipoib_upper_walk, &priv);
out:
        rcu_read_unlock();
        return data.result;
}

/* returns the number of IPoIB netdevs on top a given ipoib device matching a
 * pkey_index and address, if one exists.
 *
 * @found_net_dev: contains a matching net_device if the return value >= 1,
 * with a reference held. */
static int ipoib_match_gid_pkey_addr(struct ipoib_dev_priv *priv,
                                     const union ib_gid *gid,
                                     u16 pkey_index,
                                     const struct sockaddr *addr,
                                     int nesting,
                                     struct net_device **found_net_dev)
{
        struct ipoib_dev_priv *child_priv;
        struct net_device *net_dev = NULL;
        int matches = 0;

        if (priv->pkey_index == pkey_index &&
            (!gid || !memcmp(gid, &priv->local_gid, sizeof(*gid)))) {
                if (!addr) {
                        net_dev = ipoib_get_master_net_dev(priv->dev);
                } else {
                        /* Verify the net_device matches the IP address, as
                         * IPoIB child devices currently share a GID. */
                        net_dev = ipoib_get_net_dev_match_addr(addr, priv->dev);
                }
                if (net_dev) {
                        if (!*found_net_dev)
                                *found_net_dev = net_dev;
                        else
                                dev_put(net_dev);
                        ++matches;
                }
        }

        if (test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
                return matches;

        /* Check child interfaces */
        netdev_lock(priv->dev);
        list_for_each_entry(child_priv, &priv->child_intfs, list) {
                matches += ipoib_match_gid_pkey_addr(child_priv, gid,
                                                     pkey_index, addr,
                                                     nesting + 1,
                                                     found_net_dev);
                if (matches > 1)
                        break;
        }
        netdev_unlock(priv->dev);

        return matches;
}

/* Returns the number of matching net_devs found (between 0 and 2). Also
 * return the matching net_device in the @net_dev parameter, holding a
 * reference to the net_device, if the number of matches >= 1 */
static int __ipoib_get_net_dev_by_params(struct list_head *dev_list, u32 port,
                                         u16 pkey_index,
                                         const union ib_gid *gid,
                                         const struct sockaddr *addr,
                                         struct net_device **net_dev)
{
        struct ipoib_dev_priv *priv;
        int matches = 0;

        *net_dev = NULL;

        list_for_each_entry(priv, dev_list, list) {
                if (priv->port != port)
                        continue;

                matches += ipoib_match_gid_pkey_addr(priv, gid, pkey_index,
                                                     addr, 0, net_dev);
                if (matches > 1)
                        break;
        }

        return matches;
}

static struct net_device *ipoib_get_net_dev_by_params(
                struct ib_device *dev, u32 port, u16 pkey,
                const union ib_gid *gid, const struct sockaddr *addr,
                void *client_data)
{
        struct net_device *net_dev;
        struct list_head *dev_list = client_data;
        u16 pkey_index;
        int matches;
        int ret;

        if (!rdma_protocol_ib(dev, port))
                return NULL;

        ret = ib_find_cached_pkey(dev, port, pkey, &pkey_index);
        if (ret)
                return NULL;

        /* See if we can find a unique device matching the pkey and GID */
        matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
                                                gid, NULL, &net_dev);

        switch (matches) {
        case 0:
                return NULL;
        case 1:
                return net_dev;
        }

        dev_put(net_dev);

        /* Couldn't find a unique device with pkey and GID only. Use L3
         * address to uniquely match the net device */
        matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
                                                gid, addr, &net_dev);
        switch (matches) {
        case 0:
                return NULL;
        default:
                dev_warn_ratelimited(&dev->dev,
                                     "duplicate IP address detected\n");
                fallthrough;
        case 1:
                return net_dev;
        }
}

int ipoib_set_mode(struct net_device *dev, const char *buf)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if ((test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
             !strcmp(buf, "connected\n")) ||
             (!test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
             !strcmp(buf, "datagram\n"))) {
                return 0;
        }

        /* flush paths if we switch modes so that connections are restarted */
        if (IPOIB_CM_SUPPORTED(dev->dev_addr) && !strcmp(buf, "connected\n")) {
                set_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
                ipoib_warn(priv, "enabling connected mode "
                           "will cause multicast packet drops\n");
                netdev_lock_ops(dev);
                netdev_update_features(dev);
                netif_set_mtu(dev, ipoib_cm_max_mtu(dev));
                netif_set_real_num_tx_queues(dev, 1);
                netdev_unlock_ops(dev);
                rtnl_unlock();
                priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;

                ipoib_flush_paths(dev);
                return (!rtnl_trylock()) ? -EBUSY : 0;
        }

        if (!strcmp(buf, "datagram\n")) {
                clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
                netdev_lock_ops(dev);
                netdev_update_features(dev);
                netif_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));
                netif_set_real_num_tx_queues(dev, dev->num_tx_queues);
                netdev_unlock_ops(dev);
                rtnl_unlock();
                ipoib_flush_paths(dev);
                return (!rtnl_trylock()) ? -EBUSY : 0;
        }

        return -EINVAL;
}

struct ipoib_path *__path_find(struct net_device *dev, void *gid)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rb_node *n = priv->path_tree.rb_node;
        struct ipoib_path *path;
        int ret;

        while (n) {
                path = rb_entry(n, struct ipoib_path, rb_node);

                ret = memcmp(gid, path->pathrec.dgid.raw,
                             sizeof (union ib_gid));

                if (ret < 0)
                        n = n->rb_left;
                else if (ret > 0)
                        n = n->rb_right;
                else
                        return path;
        }

        return NULL;
}

static int __path_add(struct net_device *dev, struct ipoib_path *path)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rb_node **n = &priv->path_tree.rb_node;
        struct rb_node *pn = NULL;
        struct ipoib_path *tpath;
        int ret;

        while (*n) {
                pn = *n;
                tpath = rb_entry(pn, struct ipoib_path, rb_node);

                ret = memcmp(path->pathrec.dgid.raw, tpath->pathrec.dgid.raw,
                             sizeof (union ib_gid));
                if (ret < 0)
                        n = &pn->rb_left;
                else if (ret > 0)
                        n = &pn->rb_right;
                else
                        return -EEXIST;
        }

        rb_link_node(&path->rb_node, pn, n);
        rb_insert_color(&path->rb_node, &priv->path_tree);

        list_add_tail(&path->list, &priv->path_list);

        return 0;
}

static void path_free(struct net_device *dev, struct ipoib_path *path)
{
        struct sk_buff *skb;

        while ((skb = __skb_dequeue(&path->queue)))
                dev_kfree_skb_irq(skb);

        ipoib_dbg(ipoib_priv(dev), "%s\n", __func__);

        /* remove all neigh connected to this path */
        ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);

        if (path->ah)
                ipoib_put_ah(path->ah);

        kfree(path);
}

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG

struct ipoib_path_iter *ipoib_path_iter_init(struct net_device *dev)
{
        struct ipoib_path_iter *iter;

        iter = kmalloc_obj(*iter);
        if (!iter)
                return NULL;

        iter->dev = dev;
        memset(iter->path.pathrec.dgid.raw, 0, 16);

        if (ipoib_path_iter_next(iter)) {
                kfree(iter);
                return NULL;
        }

        return iter;
}

int ipoib_path_iter_next(struct ipoib_path_iter *iter)
{
        struct ipoib_dev_priv *priv = ipoib_priv(iter->dev);
        struct rb_node *n;
        struct ipoib_path *path;
        int ret = 1;

        spin_lock_irq(&priv->lock);

        n = rb_first(&priv->path_tree);

        while (n) {
                path = rb_entry(n, struct ipoib_path, rb_node);

                if (memcmp(iter->path.pathrec.dgid.raw, path->pathrec.dgid.raw,
                           sizeof (union ib_gid)) < 0) {
                        iter->path = *path;
                        ret = 0;
                        break;
                }

                n = rb_next(n);
        }

        spin_unlock_irq(&priv->lock);

        return ret;
}

void ipoib_path_iter_read(struct ipoib_path_iter *iter,
                          struct ipoib_path *path)
{
        *path = iter->path;
}

#endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */

void ipoib_mark_paths_invalid(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_path *path, *tp;

        spin_lock_irq(&priv->lock);

        list_for_each_entry_safe(path, tp, &priv->path_list, list) {
                ipoib_dbg(priv, "mark path LID 0x%08x GID %pI6 invalid\n",
                          be32_to_cpu(sa_path_get_dlid(&path->pathrec)),
                          path->pathrec.dgid.raw);
                if (path->ah)
                        path->ah->valid = 0;
        }

        spin_unlock_irq(&priv->lock);
}

static void push_pseudo_header(struct sk_buff *skb, const char *daddr)
{
        struct ipoib_pseudo_header *phdr;

        phdr = skb_push(skb, sizeof(*phdr));
        memcpy(phdr->hwaddr, daddr, INFINIBAND_ALEN);
}

void ipoib_flush_paths(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_path *path, *tp;
        LIST_HEAD(remove_list);
        unsigned long flags;

        netif_tx_lock_bh(dev);
        spin_lock_irqsave(&priv->lock, flags);

        list_splice_init(&priv->path_list, &remove_list);

        list_for_each_entry(path, &remove_list, list)
                rb_erase(&path->rb_node, &priv->path_tree);

        list_for_each_entry_safe(path, tp, &remove_list, list) {
                if (path->query)
                        ib_sa_cancel_query(path->query_id, path->query);
                spin_unlock_irqrestore(&priv->lock, flags);
                netif_tx_unlock_bh(dev);
                wait_for_completion(&path->done);
                path_free(dev, path);
                netif_tx_lock_bh(dev);
                spin_lock_irqsave(&priv->lock, flags);
        }

        spin_unlock_irqrestore(&priv->lock, flags);
        netif_tx_unlock_bh(dev);
}

static void path_rec_completion(int status,
                                struct sa_path_rec *pathrec,
                                unsigned int num_prs, void *path_ptr)
{
        struct ipoib_path *path = path_ptr;
        struct net_device *dev = path->dev;
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_ah *ah = NULL;
        struct ipoib_ah *old_ah = NULL;
        struct ipoib_neigh *neigh, *tn;
        struct sk_buff_head skqueue;
        struct sk_buff *skb;
        unsigned long flags;

        if (!status)
                ipoib_dbg(priv, "PathRec LID 0x%04x for GID %pI6\n",
                          be32_to_cpu(sa_path_get_dlid(pathrec)),
                          pathrec->dgid.raw);
        else
                ipoib_dbg(priv, "PathRec status %d for GID %pI6\n",
                          status, path->pathrec.dgid.raw);

        skb_queue_head_init(&skqueue);

        if (!status) {
                struct rdma_ah_attr av;

                if (!ib_init_ah_attr_from_path(priv->ca, priv->port,
                                               pathrec, &av, NULL)) {
                        ah = ipoib_create_ah(dev, priv->pd, &av);
                        rdma_destroy_ah_attr(&av);
                }
        }

        spin_lock_irqsave(&priv->lock, flags);

        if (!IS_ERR_OR_NULL(ah)) {
                /*
                 * pathrec.dgid is used as the database key from the LLADDR,
                 * it must remain unchanged even if the SA returns a different
                 * GID to use in the AH.
                 */
                if (memcmp(pathrec->dgid.raw, path->pathrec.dgid.raw,
                           sizeof(union ib_gid))) {
                        ipoib_dbg(
                                priv,
                                "%s got PathRec for gid %pI6 while asked for %pI6\n",
                                dev->name, pathrec->dgid.raw,
                                path->pathrec.dgid.raw);
                        memcpy(pathrec->dgid.raw, path->pathrec.dgid.raw,
                               sizeof(union ib_gid));
                }

                path->pathrec = *pathrec;

                old_ah   = path->ah;
                path->ah = ah;

                ipoib_dbg(priv, "created address handle %p for LID 0x%04x, SL %d\n",
                          ah, be32_to_cpu(sa_path_get_dlid(pathrec)),
                          pathrec->sl);

                while ((skb = __skb_dequeue(&path->queue)))
                        __skb_queue_tail(&skqueue, skb);

                list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {
                        if (neigh->ah) {
                                WARN_ON(neigh->ah != old_ah);
                                /*
                                 * Dropping the ah reference inside
                                 * priv->lock is safe here, because we
                                 * will hold one more reference from
                                 * the original value of path->ah (ie
                                 * old_ah).
                                 */
                                ipoib_put_ah(neigh->ah);
                        }
                        kref_get(&path->ah->ref);
                        neigh->ah = path->ah;

                        if (ipoib_cm_enabled(dev, neigh->daddr)) {
                                if (!ipoib_cm_get(neigh))
                                        ipoib_cm_set(neigh, ipoib_cm_create_tx(dev,
                                                                               path,
                                                                               neigh));
                                if (!ipoib_cm_get(neigh)) {
                                        ipoib_neigh_free(neigh);
                                        continue;
                                }
                        }

                        while ((skb = __skb_dequeue(&neigh->queue)))
                                __skb_queue_tail(&skqueue, skb);
                }
                path->ah->valid = 1;
        }

        path->query = NULL;
        complete(&path->done);

        spin_unlock_irqrestore(&priv->lock, flags);

        if (IS_ERR_OR_NULL(ah))
                ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);

        if (old_ah)
                ipoib_put_ah(old_ah);

        while ((skb = __skb_dequeue(&skqueue))) {
                int ret;
                skb->dev = dev;
                ret = dev_queue_xmit(skb);
                if (ret)
                        ipoib_warn(priv, "%s: dev_queue_xmit failed to re-queue packet, ret:%d\n",
                                   __func__, ret);
        }
}

static void init_path_rec(struct ipoib_dev_priv *priv, struct ipoib_path *path,
                          void *gid)
{
        path->dev = priv->dev;

        if (rdma_cap_opa_ah(priv->ca, priv->port))
                path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA;
        else
                path->pathrec.rec_type = SA_PATH_REC_TYPE_IB;

        memcpy(path->pathrec.dgid.raw, gid, sizeof(union ib_gid));
        path->pathrec.sgid            = priv->local_gid;
        path->pathrec.pkey            = cpu_to_be16(priv->pkey);
        path->pathrec.numb_path     = 1;
        path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class;
}

static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_path *path;

        if (!priv->broadcast)
                return NULL;

        path = kzalloc_obj(*path, GFP_ATOMIC);
        if (!path)
                return NULL;

        skb_queue_head_init(&path->queue);

        INIT_LIST_HEAD(&path->neigh_list);

        init_path_rec(priv, path, gid);

        return path;
}

static int path_rec_start(struct net_device *dev,
                          struct ipoib_path *path)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_dbg(priv, "Start path record lookup for %pI6\n",
                  path->pathrec.dgid.raw);

        init_completion(&path->done);

        path->query_id =
                ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,
                                   &path->pathrec,
                                   IB_SA_PATH_REC_DGID                |
                                   IB_SA_PATH_REC_SGID                |
                                   IB_SA_PATH_REC_NUMB_PATH        |
                                   IB_SA_PATH_REC_TRAFFIC_CLASS |
                                   IB_SA_PATH_REC_PKEY,
                                   1000, GFP_ATOMIC,
                                   path_rec_completion,
                                   path, &path->query);
        if (path->query_id < 0) {
                ipoib_warn(priv, "ib_sa_path_rec_get failed: %d\n", path->query_id);
                path->query = NULL;
                complete(&path->done);
                return path->query_id;
        }

        return 0;
}

static void neigh_refresh_path(struct ipoib_neigh *neigh, u8 *daddr,
                               struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_path *path;
        unsigned long flags;

        spin_lock_irqsave(&priv->lock, flags);

        path = __path_find(dev, daddr + 4);
        if (!path)
                goto out;
        if (!path->query)
                path_rec_start(dev, path);
out:
        spin_unlock_irqrestore(&priv->lock, flags);
}

static struct ipoib_neigh *neigh_add_path(struct sk_buff *skb, u8 *daddr,
                                          struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);
        struct ipoib_path *path;
        struct ipoib_neigh *neigh;
        unsigned long flags;

        spin_lock_irqsave(&priv->lock, flags);
        neigh = ipoib_neigh_alloc(daddr, dev);
        if (!neigh) {
                spin_unlock_irqrestore(&priv->lock, flags);
                ++dev->stats.tx_dropped;
                dev_kfree_skb_any(skb);
                return NULL;
        }

        /* To avoid race condition, make sure that the
         * neigh will be added only once.
         */
        if (unlikely(!list_empty(&neigh->list))) {
                spin_unlock_irqrestore(&priv->lock, flags);
                return neigh;
        }

        path = __path_find(dev, daddr + 4);
        if (!path) {
                path = path_rec_create(dev, daddr + 4);
                if (!path)
                        goto err_path;

                __path_add(dev, path);
        }

        list_add_tail(&neigh->list, &path->neigh_list);

        if (path->ah && path->ah->valid) {
                kref_get(&path->ah->ref);
                neigh->ah = path->ah;

                if (ipoib_cm_enabled(dev, neigh->daddr)) {
                        if (!ipoib_cm_get(neigh))
                                ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));
                        if (!ipoib_cm_get(neigh)) {
                                ipoib_neigh_free(neigh);
                                goto err_drop;
                        }
                        if (skb_queue_len(&neigh->queue) <
                            IPOIB_MAX_PATH_REC_QUEUE) {
                                push_pseudo_header(skb, neigh->daddr);
                                __skb_queue_tail(&neigh->queue, skb);
                        } else {
                                ipoib_warn(priv, "queue length limit %d. Packet drop.\n",
                                           skb_queue_len(&neigh->queue));
                                goto err_drop;
                        }
                } else {
                        spin_unlock_irqrestore(&priv->lock, flags);
                        path->ah->last_send = rn->send(dev, skb, path->ah->ah,
                                                       IPOIB_QPN(daddr));
                        ipoib_neigh_put(neigh);
                        return NULL;
                }
        } else {
                neigh->ah  = NULL;

                if (!path->query && path_rec_start(dev, path))
                        goto err_path;
                if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
                        push_pseudo_header(skb, neigh->daddr);
                        __skb_queue_tail(&neigh->queue, skb);
                } else {
                        goto err_drop;
                }
        }

        spin_unlock_irqrestore(&priv->lock, flags);
        ipoib_neigh_put(neigh);
        return NULL;

err_path:
        ipoib_neigh_free(neigh);
err_drop:
        ++dev->stats.tx_dropped;
        dev_kfree_skb_any(skb);

        spin_unlock_irqrestore(&priv->lock, flags);
        ipoib_neigh_put(neigh);

        return NULL;
}

static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
                             struct ipoib_pseudo_header *phdr)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);
        struct ipoib_path *path;
        unsigned long flags;

        spin_lock_irqsave(&priv->lock, flags);

        /* no broadcast means that all paths are (going to be) not valid */
        if (!priv->broadcast)
                goto drop_and_unlock;

        path = __path_find(dev, phdr->hwaddr + 4);
        if (!path || !path->ah || !path->ah->valid) {
                if (!path) {
                        path = path_rec_create(dev, phdr->hwaddr + 4);
                        if (!path)
                                goto drop_and_unlock;
                        __path_add(dev, path);
                } else {
                        /*
                         * make sure there are no changes in the existing
                         * path record
                         */
                        init_path_rec(priv, path, phdr->hwaddr + 4);
                }
                if (!path->query && path_rec_start(dev, path)) {
                        goto drop_and_unlock;
                }

                if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
                        push_pseudo_header(skb, phdr->hwaddr);
                        __skb_queue_tail(&path->queue, skb);
                        goto unlock;
                } else {
                        goto drop_and_unlock;
                }
        }

        spin_unlock_irqrestore(&priv->lock, flags);
        ipoib_dbg(priv, "Send unicast ARP to %08x\n",
                  be32_to_cpu(sa_path_get_dlid(&path->pathrec)));
        path->ah->last_send = rn->send(dev, skb, path->ah->ah,
                                       IPOIB_QPN(phdr->hwaddr));
        return;

drop_and_unlock:
        ++dev->stats.tx_dropped;
        dev_kfree_skb_any(skb);
unlock:
        spin_unlock_irqrestore(&priv->lock, flags);
}

static netdev_tx_t ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);
        struct ipoib_neigh *neigh;
        struct ipoib_pseudo_header *phdr;
        struct ipoib_header *header;
        unsigned long flags;

        phdr = (struct ipoib_pseudo_header *) skb->data;
        skb_pull(skb, sizeof(*phdr));
        header = (struct ipoib_header *) skb->data;

        if (unlikely(phdr->hwaddr[4] == 0xff)) {
                /* multicast, arrange "if" according to probability */
                if ((header->proto != htons(ETH_P_IP)) &&
                    (header->proto != htons(ETH_P_IPV6)) &&
                    (header->proto != htons(ETH_P_ARP)) &&
                    (header->proto != htons(ETH_P_RARP)) &&
                    (header->proto != htons(ETH_P_TIPC))) {
                        /* ethertype not supported by IPoIB */
                        ++dev->stats.tx_dropped;
                        dev_kfree_skb_any(skb);
                        return NETDEV_TX_OK;
                }
                /* Add in the P_Key for multicast*/
                phdr->hwaddr[8] = (priv->pkey >> 8) & 0xff;
                phdr->hwaddr[9] = priv->pkey & 0xff;

                neigh = ipoib_neigh_get(dev, phdr->hwaddr);
                if (likely(neigh))
                        goto send_using_neigh;
                ipoib_mcast_send(dev, phdr->hwaddr, skb);
                return NETDEV_TX_OK;
        }

        /* unicast, arrange "switch" according to probability */
        switch (header->proto) {
        case htons(ETH_P_IP):
        case htons(ETH_P_IPV6):
        case htons(ETH_P_TIPC):
                neigh = ipoib_neigh_get(dev, phdr->hwaddr);
                if (unlikely(!neigh)) {
                        neigh = neigh_add_path(skb, phdr->hwaddr, dev);
                        if (likely(!neigh))
                                return NETDEV_TX_OK;
                }
                break;
        case htons(ETH_P_ARP):
        case htons(ETH_P_RARP):
                /* for unicast ARP and RARP should always perform path find */
                unicast_arp_send(skb, dev, phdr);
                return NETDEV_TX_OK;
        default:
                /* ethertype not supported by IPoIB */
                ++dev->stats.tx_dropped;
                dev_kfree_skb_any(skb);
                return NETDEV_TX_OK;
        }

send_using_neigh:
        /* note we now hold a ref to neigh */
        if (ipoib_cm_get(neigh)) {
                if (ipoib_cm_up(neigh)) {
                        ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));
                        goto unref;
                }
        } else if (neigh->ah && neigh->ah->valid) {
                neigh->ah->last_send = rn->send(dev, skb, neigh->ah->ah,
                                                IPOIB_QPN(phdr->hwaddr));
                goto unref;
        } else if (neigh->ah) {
                neigh_refresh_path(neigh, phdr->hwaddr, dev);
        }

        if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
                push_pseudo_header(skb, phdr->hwaddr);
                spin_lock_irqsave(&priv->lock, flags);
                __skb_queue_tail(&neigh->queue, skb);
                spin_unlock_irqrestore(&priv->lock, flags);
        } else {
                ++dev->stats.tx_dropped;
                dev_kfree_skb_any(skb);
        }

unref:
        ipoib_neigh_put(neigh);

        return NETDEV_TX_OK;
}

static void ipoib_timeout(struct net_device *dev, unsigned int txqueue)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);

        if (rn->tx_timeout) {
                rn->tx_timeout(dev, txqueue);
                return;
        }
        ipoib_warn(priv, "transmit timeout: latency %d msecs\n",
                   jiffies_to_msecs(jiffies - dev_trans_start(dev)));
        ipoib_warn(priv,
                   "queue stopped %d, tx_head %u, tx_tail %u, global_tx_head %u, global_tx_tail %u\n",
                   netif_queue_stopped(dev), priv->tx_head, priv->tx_tail,
                   priv->global_tx_head, priv->global_tx_tail);


        schedule_work(&priv->tx_timeout_work);
}

void ipoib_ib_tx_timeout_work(struct work_struct *work)
{
        struct ipoib_dev_priv *priv = container_of(work,
                                                   struct ipoib_dev_priv,
                                                   tx_timeout_work);
        int err;

        rtnl_lock();
        netdev_lock_ops(priv->dev);

        if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
                goto unlock;

        ipoib_stop(priv->dev);
        err = ipoib_open(priv->dev);
        if (err) {
                ipoib_warn(priv, "ipoib_open failed recovering from a tx_timeout, err(%d).\n",
                                err);
                goto unlock;
        }

        netif_tx_wake_all_queues(priv->dev);
unlock:
        netdev_unlock_ops(priv->dev);
        rtnl_unlock();

}

static int ipoib_hard_header(struct sk_buff *skb,
                             struct net_device *dev,
                             unsigned short type,
                             const void *daddr,
                             const void *saddr,
                             unsigned int len)
{
        struct ipoib_header *header;

        header = skb_push(skb, sizeof(*header));

        header->proto = htons(type);
        header->reserved = 0;

        /*
         * we don't rely on dst_entry structure,  always stuff the
         * destination address into skb hard header so we can figure out where
         * to send the packet later.
         */
        push_pseudo_header(skb, daddr);

        return IPOIB_HARD_LEN;
}

static void ipoib_set_rx_mode_async(struct net_device *dev,
                                    struct netdev_hw_addr_list *uc,
                                    struct netdev_hw_addr_list *mc)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {
                ipoib_dbg(priv, "IPOIB_FLAG_OPER_UP not set");
                return;
        }

        queue_work(priv->wq, &priv->restart_task);
}

static int ipoib_get_iflink(const struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        /* parent interface */
        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
                return READ_ONCE(dev->ifindex);

        /* child/vlan interface */
        return READ_ONCE(priv->parent->ifindex);
}

static u32 ipoib_addr_hash(struct ipoib_neigh_hash *htbl, u8 *daddr)
{
        /*
         * Use only the address parts that contributes to spreading
         * The subnet prefix is not used as one can not connect to
         * same remote port (GUID) using the same remote QPN via two
         * different subnets.
         */
         /* qpn octets[1:4) & port GUID octets[12:20) */
        u32 *d32 = (u32 *) daddr;
        u32 hv;

        hv = jhash_3words(d32[3], d32[4], IPOIB_QPN_MASK & d32[0], 0);
        return hv & htbl->mask;
}

struct ipoib_neigh *ipoib_neigh_get(struct net_device *dev, u8 *daddr)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        struct ipoib_neigh *neigh = NULL;
        u32 hash_val;

        rcu_read_lock_bh();

        htbl = rcu_dereference_bh(ntbl->htbl);

        if (!htbl)
                goto out_unlock;

        hash_val = ipoib_addr_hash(htbl, daddr);
        for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);
             neigh != NULL;
             neigh = rcu_dereference_bh(neigh->hnext)) {
                if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
                        /* found, take one ref on behalf of the caller */
                        if (!refcount_inc_not_zero(&neigh->refcnt)) {
                                /* deleted */
                                neigh = NULL;
                                goto out_unlock;
                        }

                        if (likely(skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE))
                                neigh->alive = jiffies;
                        goto out_unlock;
                }
        }

out_unlock:
        rcu_read_unlock_bh();
        return neigh;
}

static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
{
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        unsigned long neigh_obsolete;
        unsigned long dt;
        unsigned long flags;
        int i;
        LIST_HEAD(remove_list);

        spin_lock_irqsave(&priv->lock, flags);

        htbl = rcu_dereference_protected(ntbl->htbl,
                                         lockdep_is_held(&priv->lock));

        if (!htbl)
                goto out_unlock;

        /* neigh is obsolete if it was idle for two GC periods */
        dt = 2 * arp_tbl.gc_interval;
        neigh_obsolete = jiffies - dt;

        for (i = 0; i < htbl->size; i++) {
                struct ipoib_neigh *neigh;
                struct ipoib_neigh __rcu **np = &htbl->buckets[i];

                while ((neigh = rcu_dereference_protected(*np,
                                                          lockdep_is_held(&priv->lock))) != NULL) {
                        /* was the neigh idle for two GC periods */
                        if (time_after(neigh_obsolete, neigh->alive)) {

                                ipoib_check_and_add_mcast_sendonly(priv, neigh->daddr + 4, &remove_list);

                                rcu_assign_pointer(*np,
                                                   rcu_dereference_protected(neigh->hnext,
                                                                             lockdep_is_held(&priv->lock)));
                                /* remove from path/mc list */
                                list_del_init(&neigh->list);
                                call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
                        } else {
                                np = &neigh->hnext;
                        }

                }
        }

out_unlock:
        spin_unlock_irqrestore(&priv->lock, flags);
        ipoib_mcast_remove_list(&remove_list);
}

static void ipoib_reap_neigh(struct work_struct *work)
{
        struct ipoib_dev_priv *priv =
                container_of(work, struct ipoib_dev_priv, neigh_reap_task.work);

        __ipoib_reap_neigh(priv);

        queue_delayed_work(priv->wq, &priv->neigh_reap_task,
                           arp_tbl.gc_interval);
}


static struct ipoib_neigh *ipoib_neigh_ctor(u8 *daddr,
                                      struct net_device *dev)
{
        struct ipoib_neigh *neigh;

        neigh = kzalloc_obj(*neigh, GFP_ATOMIC);
        if (!neigh)
                return NULL;

        neigh->dev = dev;
        memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));
        skb_queue_head_init(&neigh->queue);
        INIT_LIST_HEAD(&neigh->list);
        ipoib_cm_set(neigh, NULL);
        /* one ref on behalf of the caller */
        refcount_set(&neigh->refcnt, 1);

        return neigh;
}

struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr,
                                      struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        struct ipoib_neigh *neigh;
        u32 hash_val;

        htbl = rcu_dereference_protected(ntbl->htbl,
                                         lockdep_is_held(&priv->lock));
        if (!htbl) {
                neigh = NULL;
                goto out_unlock;
        }

        /* need to add a new neigh, but maybe some other thread succeeded?
         * recalc hash, maybe hash resize took place so we do a search
         */
        hash_val = ipoib_addr_hash(htbl, daddr);
        for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],
                                               lockdep_is_held(&priv->lock));
             neigh != NULL;
             neigh = rcu_dereference_protected(neigh->hnext,
                                               lockdep_is_held(&priv->lock))) {
                if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
                        /* found, take one ref on behalf of the caller */
                        if (!refcount_inc_not_zero(&neigh->refcnt)) {
                                /* deleted */
                                neigh = NULL;
                                break;
                        }
                        neigh->alive = jiffies;
                        goto out_unlock;
                }
        }

        neigh = ipoib_neigh_ctor(daddr, dev);
        if (!neigh)
                goto out_unlock;

        /* one ref on behalf of the hash table */
        refcount_inc(&neigh->refcnt);
        neigh->alive = jiffies;
        /* put in hash */
        rcu_assign_pointer(neigh->hnext,
                           rcu_dereference_protected(htbl->buckets[hash_val],
                                                     lockdep_is_held(&priv->lock)));
        rcu_assign_pointer(htbl->buckets[hash_val], neigh);
        atomic_inc(&ntbl->entries);

out_unlock:

        return neigh;
}

void ipoib_neigh_dtor(struct ipoib_neigh *neigh)
{
        /* neigh reference count was dropprd to zero */
        struct net_device *dev = neigh->dev;
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct sk_buff *skb;
        if (neigh->ah)
                ipoib_put_ah(neigh->ah);
        while ((skb = __skb_dequeue(&neigh->queue))) {
                ++dev->stats.tx_dropped;
                dev_kfree_skb_any(skb);
        }
        if (ipoib_cm_get(neigh))
                ipoib_cm_destroy_tx(ipoib_cm_get(neigh));
        ipoib_dbg(ipoib_priv(dev),
                  "neigh free for %06x %pI6\n",
                  IPOIB_QPN(neigh->daddr),
                  neigh->daddr + 4);
        kfree(neigh);
        if (atomic_dec_and_test(&priv->ntbl.entries)) {
                if (test_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags))
                        complete(&priv->ntbl.flushed);
        }
}

static void ipoib_neigh_reclaim(struct rcu_head *rp)
{
        /* Called as a result of removal from hash table */
        struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);
        /* note TX context may hold another ref */
        ipoib_neigh_put(neigh);
}

void ipoib_neigh_free(struct ipoib_neigh *neigh)
{
        struct net_device *dev = neigh->dev;
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        struct ipoib_neigh __rcu **np;
        struct ipoib_neigh *n;
        u32 hash_val;

        htbl = rcu_dereference_protected(ntbl->htbl,
                                        lockdep_is_held(&priv->lock));
        if (!htbl)
                return;

        hash_val = ipoib_addr_hash(htbl, neigh->daddr);
        np = &htbl->buckets[hash_val];
        for (n = rcu_dereference_protected(*np,
                                            lockdep_is_held(&priv->lock));
             n != NULL;
             n = rcu_dereference_protected(*np,
                                        lockdep_is_held(&priv->lock))) {
                if (n == neigh) {
                        /* found */
                        rcu_assign_pointer(*np,
                                           rcu_dereference_protected(neigh->hnext,
                                                                     lockdep_is_held(&priv->lock)));
                        /* remove from parent list */
                        list_del_init(&neigh->list);
                        call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
                        return;
                } else {
                        np = &n->hnext;
                }
        }
}

static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)
{
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        struct ipoib_neigh __rcu **buckets;
        u32 size;

        clear_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);
        ntbl->htbl = NULL;
        htbl = kzalloc_obj(*htbl);
        if (!htbl)
                return -ENOMEM;
        size = roundup_pow_of_two(arp_tbl.gc_thresh3);
        buckets = kvzalloc_objs(*buckets, size);
        if (!buckets) {
                kfree(htbl);
                return -ENOMEM;
        }
        htbl->size = size;
        htbl->mask = (size - 1);
        htbl->buckets = buckets;
        RCU_INIT_POINTER(ntbl->htbl, htbl);
        htbl->ntbl = ntbl;
        atomic_set(&ntbl->entries, 0);

        /* start garbage collection */
        queue_delayed_work(priv->wq, &priv->neigh_reap_task,
                           arp_tbl.gc_interval);

        return 0;
}

static void neigh_hash_free_rcu(struct rcu_head *head)
{
        struct ipoib_neigh_hash *htbl = container_of(head,
                                                    struct ipoib_neigh_hash,
                                                    rcu);
        struct ipoib_neigh __rcu **buckets = htbl->buckets;
        struct ipoib_neigh_table *ntbl = htbl->ntbl;

        kvfree(buckets);
        kfree(htbl);
        complete(&ntbl->deleted);
}

void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        unsigned long flags;
        int i;

        /* remove all neigh connected to a given path or mcast */
        spin_lock_irqsave(&priv->lock, flags);

        htbl = rcu_dereference_protected(ntbl->htbl,
                                         lockdep_is_held(&priv->lock));

        if (!htbl)
                goto out_unlock;

        for (i = 0; i < htbl->size; i++) {
                struct ipoib_neigh *neigh;
                struct ipoib_neigh __rcu **np = &htbl->buckets[i];

                while ((neigh = rcu_dereference_protected(*np,
                                                          lockdep_is_held(&priv->lock))) != NULL) {
                        /* delete neighs belong to this parent */
                        if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) {
                                rcu_assign_pointer(*np,
                                                   rcu_dereference_protected(neigh->hnext,
                                                                             lockdep_is_held(&priv->lock)));
                                /* remove from parent list */
                                list_del_init(&neigh->list);
                                call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
                        } else {
                                np = &neigh->hnext;
                        }

                }
        }
out_unlock:
        spin_unlock_irqrestore(&priv->lock, flags);
}

static void ipoib_flush_neighs(struct ipoib_dev_priv *priv)
{
        struct ipoib_neigh_table *ntbl = &priv->ntbl;
        struct ipoib_neigh_hash *htbl;
        unsigned long flags;
        int i, wait_flushed = 0;

        init_completion(&priv->ntbl.flushed);
        set_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);

        spin_lock_irqsave(&priv->lock, flags);

        htbl = rcu_dereference_protected(ntbl->htbl,
                                        lockdep_is_held(&priv->lock));
        if (!htbl)
                goto out_unlock;

        wait_flushed = atomic_read(&priv->ntbl.entries);
        if (!wait_flushed)
                goto free_htbl;

        for (i = 0; i < htbl->size; i++) {
                struct ipoib_neigh *neigh;
                struct ipoib_neigh __rcu **np = &htbl->buckets[i];

                while ((neigh = rcu_dereference_protected(*np,
                                       lockdep_is_held(&priv->lock))) != NULL) {
                        rcu_assign_pointer(*np,
                                           rcu_dereference_protected(neigh->hnext,
                                                                     lockdep_is_held(&priv->lock)));
                        /* remove from path/mc list */
                        list_del_init(&neigh->list);
                        call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
                }
        }

free_htbl:
        rcu_assign_pointer(ntbl->htbl, NULL);
        call_rcu(&htbl->rcu, neigh_hash_free_rcu);

out_unlock:
        spin_unlock_irqrestore(&priv->lock, flags);
        if (wait_flushed)
                wait_for_completion(&priv->ntbl.flushed);
}

static void ipoib_neigh_hash_uninit(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_dbg(priv, "%s\n", __func__);
        init_completion(&priv->ntbl.deleted);

        cancel_delayed_work_sync(&priv->neigh_reap_task);

        ipoib_flush_neighs(priv);

        wait_for_completion(&priv->ntbl.deleted);
}

static void ipoib_napi_add(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        netif_napi_add_weight(dev, &priv->recv_napi, ipoib_rx_poll,
                              IPOIB_NUM_WC);
        netif_napi_add_weight(dev, &priv->send_napi, ipoib_tx_poll,
                              MAX_SEND_CQE);
}

static void ipoib_napi_del(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        netif_napi_del(&priv->recv_napi);
        netif_napi_del(&priv->send_napi);
}

static void ipoib_dev_uninit_default(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        ipoib_transport_dev_cleanup(dev);

        ipoib_napi_del(dev);

        ipoib_cm_dev_cleanup(dev);

        kfree(priv->rx_ring);
        vfree(priv->tx_ring);

        priv->rx_ring = NULL;
        priv->tx_ring = NULL;
}

static int ipoib_dev_init_default(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        u8 addr_mod[3];

        ipoib_napi_add(dev);

        /* Allocate RX/TX "rings" to hold queued skbs */
        priv->rx_ring =        kzalloc_objs(*priv->rx_ring, ipoib_recvq_size);
        if (!priv->rx_ring)
                goto out;

        priv->tx_ring = vzalloc(array_size(ipoib_sendq_size,
                                           sizeof(*priv->tx_ring)));
        if (!priv->tx_ring) {
                pr_warn("%s: failed to allocate TX ring (%d entries)\n",
                        priv->ca->name, ipoib_sendq_size);
                goto out_rx_ring_cleanup;
        }

        /* priv->tx_head, tx_tail and global_tx_tail/head are already 0 */

        if (ipoib_transport_dev_init(dev, priv->ca)) {
                pr_warn("%s: ipoib_transport_dev_init failed\n",
                        priv->ca->name);
                goto out_tx_ring_cleanup;
        }

        /* after qp created set dev address */
        addr_mod[0] = (priv->qp->qp_num >> 16) & 0xff;
        addr_mod[1] = (priv->qp->qp_num >>  8) & 0xff;
        addr_mod[2] = (priv->qp->qp_num) & 0xff;
        dev_addr_mod(priv->dev, 1, addr_mod, sizeof(addr_mod));

        return 0;

out_tx_ring_cleanup:
        vfree(priv->tx_ring);

out_rx_ring_cleanup:
        kfree(priv->rx_ring);

out:
        ipoib_napi_del(dev);
        return -ENOMEM;
}

static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
                       int cmd)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (!priv->rn_ops->ndo_eth_ioctl)
                return -EOPNOTSUPP;

        return priv->rn_ops->ndo_eth_ioctl(dev, ifr, cmd);
}

static int ipoib_hwtstamp_get(struct net_device *dev,
                              struct kernel_hwtstamp_config *config)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (!priv->rn_ops->ndo_hwtstamp_get)
                return -EOPNOTSUPP;

        return priv->rn_ops->ndo_hwtstamp_get(dev, config);
}

static int ipoib_hwtstamp_set(struct net_device *dev,
                              struct kernel_hwtstamp_config *config,
                              struct netlink_ext_ack *extack)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (!priv->rn_ops->ndo_hwtstamp_set)
                return -EOPNOTSUPP;

        return priv->rn_ops->ndo_hwtstamp_set(dev, config, extack);
}

static int ipoib_dev_init(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        int ret = -ENOMEM;

        priv->qp = NULL;

        /*
         * the various IPoIB tasks assume they will never race against
         * themselves, so always use a single thread workqueue
         */
        priv->wq = alloc_ordered_workqueue("ipoib_wq", WQ_MEM_RECLAIM);
        if (!priv->wq) {
                pr_warn("%s: failed to allocate device WQ\n", dev->name);
                goto out;
        }

        /* create pd, which used both for control and datapath*/
        priv->pd = ib_alloc_pd(priv->ca, 0);
        if (IS_ERR(priv->pd)) {
                pr_warn("%s: failed to allocate PD\n", priv->ca->name);
                goto clean_wq;
        }

        ret = priv->rn_ops->ndo_init(dev);
        if (ret) {
                pr_warn("%s failed to init HW resource\n", dev->name);
                goto out_free_pd;
        }

        ret = ipoib_neigh_hash_init(priv);
        if (ret) {
                pr_warn("%s failed to init neigh hash\n", dev->name);
                goto out_dev_uninit;
        }

        if (dev->flags & IFF_UP) {
                if (ipoib_ib_dev_open(dev)) {
                        pr_warn("%s failed to open device\n", dev->name);
                        ret = -ENODEV;
                        goto out_hash_uninit;
                }
        }

        return 0;

out_hash_uninit:
        ipoib_neigh_hash_uninit(dev);

out_dev_uninit:
        ipoib_ib_dev_cleanup(dev);

out_free_pd:
        if (priv->pd) {
                ib_dealloc_pd(priv->pd);
                priv->pd = NULL;
        }

clean_wq:
        if (priv->wq) {
                destroy_workqueue(priv->wq);
                priv->wq = NULL;
        }

out:
        return ret;
}

/*
 * This must be called before doing an unregister_netdev on a parent device to
 * shutdown the IB event handler.
 */
static void ipoib_parent_unregister_pre(struct net_device *ndev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);

        /*
         * ipoib_set_mac checks netif_running before pushing work, clearing
         * running ensures the it will not add more work.
         */
        rtnl_lock();
        dev_change_flags(priv->dev, priv->dev->flags & ~IFF_UP, NULL);
        rtnl_unlock();

        /* ipoib_event() cannot be running once this returns */
        ib_unregister_event_handler(&priv->event_handler);

        /*
         * Work on the queue grabs the rtnl lock, so this cannot be done while
         * also holding it.
         */
        flush_workqueue(ipoib_workqueue);
}

static void ipoib_set_dev_features(struct ipoib_dev_priv *priv)
{
        priv->hca_caps = priv->ca->attrs.device_cap_flags;
        priv->kernel_caps = priv->ca->attrs.kernel_cap_flags;

        if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) {
                priv->dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_RXCSUM;

                if (priv->kernel_caps & IBK_UD_TSO)
                        priv->dev->hw_features |= NETIF_F_TSO;

                priv->dev->features |= priv->dev->hw_features;
        }
}

static int ipoib_parent_init(struct net_device *ndev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
        struct ib_port_attr attr;
        int result;

        result = ib_query_port(priv->ca, priv->port, &attr);
        if (result) {
                pr_warn("%s: ib_query_port %d failed\n", priv->ca->name,
                        priv->port);
                return result;
        }
        priv->max_ib_mtu = rdma_mtu_from_attr(priv->ca, priv->port, &attr);

        result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);
        if (result) {
                pr_warn("%s: ib_query_pkey port %d failed (ret = %d)\n",
                        priv->ca->name, priv->port, result);
                return result;
        }

        result = rdma_query_gid(priv->ca, priv->port, 0, &priv->local_gid);
        if (result) {
                pr_warn("%s: rdma_query_gid port %d failed (ret = %d)\n",
                        priv->ca->name, priv->port, result);
                return result;
        }
        dev_addr_mod(priv->dev, 4, priv->local_gid.raw, sizeof(union ib_gid));

        SET_NETDEV_DEV(priv->dev, priv->ca->dev.parent);
        priv->dev->dev_port = priv->port - 1;
        /* Let's set this one too for backwards compatibility. */
        priv->dev->dev_id = priv->port - 1;

        return 0;
}

static void ipoib_child_init(struct net_device *ndev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
        struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);

        priv->max_ib_mtu = ppriv->max_ib_mtu;
        set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags);
        if (memchr_inv(priv->dev->dev_addr, 0, INFINIBAND_ALEN))
                memcpy(&priv->local_gid, priv->dev->dev_addr + 4,
                       sizeof(priv->local_gid));
        else {
                __dev_addr_set(priv->dev, ppriv->dev->dev_addr,
                               INFINIBAND_ALEN);
                memcpy(&priv->local_gid, &ppriv->local_gid,
                       sizeof(priv->local_gid));
        }
}

static int ipoib_ndo_init(struct net_device *ndev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);
        int rc;
        struct rdma_netdev *rn = netdev_priv(ndev);

        if (priv->parent) {
                ipoib_child_init(ndev);
        } else {
                rc = ipoib_parent_init(ndev);
                if (rc)
                        return rc;
        }

        /* MTU will be reset when mcast join happens */
        ndev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu);
        priv->mcast_mtu = priv->admin_mtu = ndev->mtu;
        rn->mtu = priv->mcast_mtu;
        ndev->max_mtu = IPOIB_CM_MTU;

        ndev->neigh_priv_len = sizeof(struct ipoib_neigh);

        /*
         * Set the full membership bit, so that we join the right
         * broadcast group, etc.
         */
        priv->pkey |= 0x8000;

        ndev->broadcast[8] = priv->pkey >> 8;
        ndev->broadcast[9] = priv->pkey & 0xff;
        set_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);

        ipoib_set_dev_features(priv);

        rc = ipoib_dev_init(ndev);
        if (rc) {
                pr_warn("%s: failed to initialize device: %s port %d (ret = %d)\n",
                        priv->ca->name, priv->dev->name, priv->port, rc);
                return rc;
        }

        if (priv->parent) {
                struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);

                dev_hold(priv->parent);

                netdev_lock(priv->parent);
                list_add_tail(&priv->list, &ppriv->child_intfs);
                netdev_unlock(priv->parent);
        }

        return 0;
}

static void ipoib_ndo_uninit(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        /*
         * ipoib_remove_one guarantees the children are removed before the
         * parent, and that is the only place where a parent can be removed.
         */
        WARN_ON(!list_empty(&priv->child_intfs));

        if (priv->parent) {
                struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);

                netdev_lock(ppriv->dev);
                list_del(&priv->list);
                netdev_unlock(ppriv->dev);
        }

        ipoib_neigh_hash_uninit(dev);

        ipoib_ib_dev_cleanup(dev);

        /* no more works over the priv->wq */
        if (priv->wq) {
                /* See ipoib_mcast_carrier_on_task() */
                WARN_ON(test_bit(IPOIB_FLAG_OPER_UP, &priv->flags));
                destroy_workqueue(priv->wq);
                priv->wq = NULL;
        }

        dev_put(priv->parent);
}

static int ipoib_set_vf_link_state(struct net_device *dev, int vf, int link_state)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        return ib_set_vf_link_state(priv->ca, vf, priv->port, link_state);
}

static int ipoib_get_vf_config(struct net_device *dev, int vf,
                               struct ifla_vf_info *ivf)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        int err;

        err = ib_get_vf_config(priv->ca, vf, priv->port, ivf);
        if (err)
                return err;

        ivf->vf = vf;
        memcpy(ivf->mac, dev->dev_addr, dev->addr_len);

        return 0;
}

static int ipoib_set_vf_guid(struct net_device *dev, int vf, u64 guid, int type)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        if (type != IFLA_VF_IB_NODE_GUID && type != IFLA_VF_IB_PORT_GUID)
                return -EINVAL;

        return ib_set_vf_guid(priv->ca, vf, priv->port, guid, type);
}

static int ipoib_get_vf_guid(struct net_device *dev, int vf,
                             struct ifla_vf_guid *node_guid,
                             struct ifla_vf_guid *port_guid)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        return ib_get_vf_guid(priv->ca, vf, priv->port, node_guid, port_guid);
}

static int ipoib_get_vf_stats(struct net_device *dev, int vf,
                              struct ifla_vf_stats *vf_stats)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        return ib_get_vf_stats(priv->ca, vf, priv->port, vf_stats);
}

static const struct header_ops ipoib_header_ops = {
        .create        = ipoib_hard_header,
};

static const struct net_device_ops ipoib_netdev_ops_pf = {
        .ndo_init                 = ipoib_ndo_init,
        .ndo_uninit                 = ipoib_ndo_uninit,
        .ndo_open                 = ipoib_open,
        .ndo_stop                 = ipoib_stop,
        .ndo_change_mtu                 = ipoib_change_mtu,
        .ndo_fix_features         = ipoib_fix_features,
        .ndo_start_xmit                 = ipoib_start_xmit,
        .ndo_tx_timeout                 = ipoib_timeout,
        .ndo_set_rx_mode_async         = ipoib_set_rx_mode_async,
        .ndo_get_iflink                 = ipoib_get_iflink,
        .ndo_set_vf_link_state         = ipoib_set_vf_link_state,
        .ndo_get_vf_config         = ipoib_get_vf_config,
        .ndo_get_vf_stats         = ipoib_get_vf_stats,
        .ndo_get_vf_guid         = ipoib_get_vf_guid,
        .ndo_set_vf_guid         = ipoib_set_vf_guid,
        .ndo_set_mac_address         = ipoib_set_mac,
        .ndo_get_stats64         = ipoib_get_stats,
        .ndo_eth_ioctl                 = ipoib_ioctl,
        .ndo_hwtstamp_get         = ipoib_hwtstamp_get,
        .ndo_hwtstamp_set         = ipoib_hwtstamp_set,
};

static const struct net_device_ops ipoib_netdev_ops_vf = {
        .ndo_init                 = ipoib_ndo_init,
        .ndo_uninit                 = ipoib_ndo_uninit,
        .ndo_open                 = ipoib_open,
        .ndo_stop                 = ipoib_stop,
        .ndo_change_mtu                 = ipoib_change_mtu,
        .ndo_fix_features         = ipoib_fix_features,
        .ndo_start_xmit                  = ipoib_start_xmit,
        .ndo_tx_timeout                 = ipoib_timeout,
        .ndo_set_rx_mode_async         = ipoib_set_rx_mode_async,
        .ndo_get_iflink                 = ipoib_get_iflink,
        .ndo_get_stats64         = ipoib_get_stats,
        .ndo_eth_ioctl                 = ipoib_ioctl,
        .ndo_hwtstamp_get         = ipoib_hwtstamp_get,
        .ndo_hwtstamp_set         = ipoib_hwtstamp_set,
};

static const struct net_device_ops ipoib_netdev_default_pf = {
        .ndo_init                 = ipoib_dev_init_default,
        .ndo_uninit                 = ipoib_dev_uninit_default,
        .ndo_open                 = ipoib_ib_dev_open_default,
        .ndo_stop                 = ipoib_ib_dev_stop_default,
};

void ipoib_setup_common(struct net_device *dev)
{
        dev->header_ops                 = &ipoib_header_ops;
        dev->netdev_ops          = &ipoib_netdev_default_pf;

        ipoib_set_ethtool_ops(dev);

        dev->watchdog_timeo         = 10 * HZ;

        dev->flags                |= IFF_BROADCAST | IFF_MULTICAST;

        dev->hard_header_len         = IPOIB_HARD_LEN;
        dev->addr_len                 = INFINIBAND_ALEN;
        dev->type                 = ARPHRD_INFINIBAND;
        dev->tx_queue_len         = DEFAULT_TX_QUEUE_LEN;
        dev->features                 = (NETIF_F_VLAN_CHALLENGED        |
                                    NETIF_F_HIGHDMA);
        netif_keep_dst(dev);

        memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN);

        /*
         * unregister_netdev always frees the netdev, we use this mode
         * consistently to unify all the various unregister paths, including
         * those connected to rtnl_link_ops which require it.
         */
        dev->needs_free_netdev = true;
}

static void ipoib_build_priv(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);

        priv->dev = dev;
        spin_lock_init(&priv->lock);
        mutex_init(&priv->mcast_mutex);

        INIT_LIST_HEAD(&priv->path_list);
        INIT_LIST_HEAD(&priv->child_intfs);
        INIT_LIST_HEAD(&priv->dead_ahs);
        INIT_LIST_HEAD(&priv->multicast_list);

        INIT_DELAYED_WORK(&priv->mcast_task,   ipoib_mcast_join_task);
        INIT_WORK(&priv->carrier_on_task, ipoib_mcast_carrier_on_task);
        INIT_WORK(&priv->reschedule_napi_work, ipoib_napi_schedule_work);
        INIT_WORK(&priv->flush_light,   ipoib_ib_dev_flush_light);
        INIT_WORK(&priv->flush_normal,   ipoib_ib_dev_flush_normal);
        INIT_WORK(&priv->flush_heavy,   ipoib_ib_dev_flush_heavy);
        INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task);
        INIT_WORK(&priv->tx_timeout_work, ipoib_ib_tx_timeout_work);
        INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah);
        INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh);
}

static struct net_device *ipoib_alloc_netdev(struct ib_device *hca, u32 port,
                                             const char *name)
{
        struct net_device *dev;

        dev = rdma_alloc_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
                                NET_NAME_UNKNOWN, ipoib_setup_common);
        if (!IS_ERR(dev) || PTR_ERR(dev) != -EOPNOTSUPP)
                return dev;

        dev = alloc_netdev(sizeof(struct rdma_netdev), name, NET_NAME_UNKNOWN,
                           ipoib_setup_common);
        if (!dev)
                return ERR_PTR(-ENOMEM);
        return dev;
}

int ipoib_intf_init(struct ib_device *hca, u32 port, const char *name,
                    struct net_device *dev)
{
        struct rdma_netdev *rn = netdev_priv(dev);
        struct ipoib_dev_priv *priv;
        int rc;

        priv = kzalloc_obj(*priv);
        if (!priv)
                return -ENOMEM;

        priv->ca = hca;
        priv->port = port;

        rc = rdma_init_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
                              NET_NAME_UNKNOWN, ipoib_setup_common, dev);
        if (rc) {
                if (rc != -EOPNOTSUPP)
                        goto out;

                rn->send = ipoib_send;
                rn->attach_mcast = ipoib_mcast_attach;
                rn->detach_mcast = ipoib_mcast_detach;
                rn->hca = hca;

                rc = netif_set_real_num_tx_queues(dev, 1);
                if (rc)
                        goto out;

                rc = netif_set_real_num_rx_queues(dev, 1);
                if (rc)
                        goto out;
        }

        priv->rn_ops = dev->netdev_ops;

        if (hca->attrs.kernel_cap_flags & IBK_VIRTUAL_FUNCTION)
                dev->netdev_ops        = &ipoib_netdev_ops_vf;
        else
                dev->netdev_ops        = &ipoib_netdev_ops_pf;

        rn->clnt_priv = priv;
        /*
         * Only the child register_netdev flows can handle priv_destructor
         * being set, so we force it to NULL here and handle manually until it
         * is safe to turn on.
         */
        priv->next_priv_destructor = dev->priv_destructor;
        dev->priv_destructor = NULL;

        ipoib_build_priv(dev);

        return 0;

out:
        kfree(priv);
        return rc;
}

struct net_device *ipoib_intf_alloc(struct ib_device *hca, u32 port,
                                    const char *name)
{
        struct net_device *dev;
        int rc;

        dev = ipoib_alloc_netdev(hca, port, name);
        if (IS_ERR(dev))
                return dev;

        rc = ipoib_intf_init(hca, port, name, dev);
        if (rc) {
                free_netdev(dev);
                return ERR_PTR(rc);
        }

        /*
         * Upon success the caller must ensure ipoib_intf_free is called or
         * register_netdevice succeed'd and priv_destructor is set to
         * ipoib_intf_free.
         */
        return dev;
}

void ipoib_intf_free(struct net_device *dev)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct rdma_netdev *rn = netdev_priv(dev);

        dev->priv_destructor = priv->next_priv_destructor;
        if (dev->priv_destructor)
                dev->priv_destructor(dev);

        /*
         * There are some error flows around register_netdev failing that may
         * attempt to call priv_destructor twice, prevent that from happening.
         */
        dev->priv_destructor = NULL;

        /* unregister/destroy is very complicated. Make bugs more obvious. */
        rn->clnt_priv = NULL;

        kfree(priv);
}

static ssize_t pkey_show(struct device *dev, struct device_attribute *attr,
                         char *buf)
{
        struct net_device *ndev = to_net_dev(dev);
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);

        return sysfs_emit(buf, "0x%04x\n", priv->pkey);
}
static DEVICE_ATTR_RO(pkey);

static ssize_t umcast_show(struct device *dev, struct device_attribute *attr,
                           char *buf)
{
        struct net_device *ndev = to_net_dev(dev);
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);

        return sysfs_emit(buf, "%d\n",
                          test_bit(IPOIB_FLAG_UMCAST, &priv->flags));
}

void ipoib_set_umcast(struct net_device *ndev, int umcast_val)
{
        struct ipoib_dev_priv *priv = ipoib_priv(ndev);

        if (umcast_val > 0) {
                set_bit(IPOIB_FLAG_UMCAST, &priv->flags);
                ipoib_warn(priv, "ignoring multicast groups joined directly "
                                "by userspace\n");
        } else
                clear_bit(IPOIB_FLAG_UMCAST, &priv->flags);
}

static ssize_t umcast_store(struct device *dev, struct device_attribute *attr,
                            const char *buf, size_t count)
{
        unsigned long umcast_val = simple_strtoul(buf, NULL, 0);

        ipoib_set_umcast(to_net_dev(dev), umcast_val);

        return count;
}
static DEVICE_ATTR_RW(umcast);

int ipoib_add_umcast_attr(struct net_device *dev)
{
        return device_create_file(&dev->dev, &dev_attr_umcast);
}

static void set_base_guid(struct ipoib_dev_priv *priv, union ib_gid *gid)
{
        struct ipoib_dev_priv *child_priv;
        struct net_device *netdev = priv->dev;

        netif_addr_lock_bh(netdev);

        memcpy(&priv->local_gid.global.interface_id,
               &gid->global.interface_id,
               sizeof(gid->global.interface_id));
        dev_addr_mod(netdev, 4, (u8 *)&priv->local_gid, sizeof(priv->local_gid));
        clear_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);

        netif_addr_unlock_bh(netdev);

        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
                netdev_lock_ops_to_full(priv->dev);
                list_for_each_entry(child_priv, &priv->child_intfs, list)
                        set_base_guid(child_priv, gid);
                netdev_unlock_full_to_ops(priv->dev);
        }
}

static int ipoib_check_lladdr(struct net_device *dev,
                              struct sockaddr_storage *ss)
{
        union ib_gid *gid = (union ib_gid *)(ss->__data + 4);
        int ret = 0;

        netif_addr_lock_bh(dev);

        /* Make sure the QPN, reserved and subnet prefix match the current
         * lladdr, it also makes sure the lladdr is unicast.
         */
        if (memcmp(dev->dev_addr, ss->__data,
                   4 + sizeof(gid->global.subnet_prefix)) ||
            gid->global.interface_id == 0)
                ret = -EINVAL;

        netif_addr_unlock_bh(dev);

        return ret;
}

static int ipoib_set_mac(struct net_device *dev, void *addr)
{
        struct ipoib_dev_priv *priv = ipoib_priv(dev);
        struct sockaddr_storage *ss = addr;
        int ret;

        if (!(dev->priv_flags & IFF_LIVE_ADDR_CHANGE) && netif_running(dev))
                return -EBUSY;

        ret = ipoib_check_lladdr(dev, ss);
        if (ret)
                return ret;

        set_base_guid(priv, (union ib_gid *)(ss->__data + 4));

        if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
                struct ipoib_dev_priv *cpriv;

                netdev_lock_ops_to_full(dev);
                list_for_each_entry(cpriv, &priv->child_intfs, list)
                        queue_work(ipoib_workqueue, &cpriv->flush_light);
                netdev_unlock_full_to_ops(dev);
        }
        queue_work(ipoib_workqueue, &priv->flush_light);

        return 0;
}

static ssize_t create_child_store(struct device *dev,
                                  struct device_attribute *attr,
                                  const char *buf, size_t count)
{
        int pkey;
        int ret;

        if (sscanf(buf, "%i", &pkey) != 1)
                return -EINVAL;

        if (pkey <= 0 || pkey > 0xffff || pkey == 0x8000)
                return -EINVAL;

        ret = ipoib_vlan_add(to_net_dev(dev), pkey);

        return ret ? ret : count;
}
static DEVICE_ATTR_WO(create_child);

static ssize_t delete_child_store(struct device *dev,
                                  struct device_attribute *attr,
                                  const char *buf, size_t count)
{
        int pkey;
        int ret;

        if (sscanf(buf, "%i", &pkey) != 1)
                return -EINVAL;

        if (pkey < 0 || pkey > 0xffff)
                return -EINVAL;

        ret = ipoib_vlan_delete(to_net_dev(dev), pkey);

        return ret ? ret : count;

}
static DEVICE_ATTR_WO(delete_child);

int ipoib_add_pkey_attr(struct net_device *dev)
{
        return device_create_file(&dev->dev, &dev_attr_pkey);
}

/*
 * We erroneously exposed the iface's port number in the dev_id
 * sysfs field long after dev_port was introduced for that purpose[1],
 * and we need to stop everyone from relying on that.
 * Let's overload the shower routine for the dev_id file here
 * to gently bring the issue up.
 *
 * [1] https://www.spinics.net/lists/netdev/msg272123.html
 */
static ssize_t dev_id_show(struct device *dev,
                           struct device_attribute *attr, char *buf)
{
        struct net_device *ndev = to_net_dev(dev);

        /*
         * ndev->dev_port will be equal to 0 in old kernel prior to commit
         * 9b8b2a323008 ("IB/ipoib: Use dev_port to expose network interface
         * port numbers") Zero was chosen as special case for user space
         * applications to fallback and query dev_id to check if it has
         * different value or not.
         *
         * Don't print warning in such scenario.
         *
         * https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c#L358
         */
        if (ndev->dev_port && ndev->dev_id == ndev->dev_port)
                netdev_info_once(ndev,
                        "\"%s\" wants to know my dev_id. Should it look at dev_port instead? See Documentation/ABI/testing/sysfs-class-net for more info.\n",
                        current->comm);

        return sysfs_emit(buf, "%#x\n", ndev->dev_id);
}
static DEVICE_ATTR_RO(dev_id);

static int ipoib_intercept_dev_id_attr(struct net_device *dev)
{
        device_remove_file(&dev->dev, &dev_attr_dev_id);
        return device_create_file(&dev->dev, &dev_attr_dev_id);
}

static struct net_device *ipoib_add_port(const char *format,
                                         struct ib_device *hca, u32 port)
{
        struct rtnl_link_ops *ops = ipoib_get_link_ops();
        struct rdma_netdev_alloc_params params;
        struct ipoib_dev_priv *priv;
        struct net_device *ndev;
        int result;

        ndev = ipoib_intf_alloc(hca, port, format);
        if (IS_ERR(ndev)) {
                pr_warn("%s, %d: ipoib_intf_alloc failed %ld\n", hca->name, port,
                        PTR_ERR(ndev));
                return ndev;
        }
        priv = ipoib_priv(ndev);

        INIT_IB_EVENT_HANDLER(&priv->event_handler,
                              priv->ca, ipoib_event);
        ib_register_event_handler(&priv->event_handler);

        /* call event handler to ensure pkey in sync */
        ipoib_queue_work(priv, IPOIB_FLUSH_HEAVY);

        ndev->rtnl_link_ops = ipoib_get_link_ops();

        dev_net_set(ndev, rdma_dev_net(hca));

        result = register_netdev(ndev);
        if (result) {
                pr_warn("%s: couldn't register ipoib port %d; error %d\n",
                        hca->name, port, result);

                ipoib_parent_unregister_pre(ndev);
                ipoib_intf_free(ndev);
                free_netdev(ndev);

                return ERR_PTR(result);
        }

        if (hca->ops.rdma_netdev_get_params) {
                int rc = hca->ops.rdma_netdev_get_params(hca, port,
                                                     RDMA_NETDEV_IPOIB,
                                                     &params);

                if (!rc && ops->priv_size < params.sizeof_priv)
                        ops->priv_size = params.sizeof_priv;
        }
        /*
         * We cannot set priv_destructor before register_netdev because we
         * need priv to be always valid during the error flow to execute
         * ipoib_parent_unregister_pre(). Instead handle it manually and only
         * enter priv_destructor mode once we are completely registered.
         */
        ndev->priv_destructor = ipoib_intf_free;

        if (ipoib_intercept_dev_id_attr(ndev))
                goto sysfs_failed;
        if (ipoib_cm_add_mode_attr(ndev))
                goto sysfs_failed;
        if (ipoib_add_pkey_attr(ndev))
                goto sysfs_failed;
        if (ipoib_add_umcast_attr(ndev))
                goto sysfs_failed;
        if (device_create_file(&ndev->dev, &dev_attr_create_child))
                goto sysfs_failed;
        if (device_create_file(&ndev->dev, &dev_attr_delete_child))
                goto sysfs_failed;

        return ndev;

sysfs_failed:
        ipoib_parent_unregister_pre(ndev);
        unregister_netdev(ndev);
        return ERR_PTR(-ENOMEM);
}

static int ipoib_add_one(struct ib_device *device)
{
        struct list_head *dev_list;
        struct net_device *dev;
        struct ipoib_dev_priv *priv;
        unsigned int p;
        int count = 0;

        dev_list = kmalloc_obj(*dev_list);
        if (!dev_list)
                return -ENOMEM;

        INIT_LIST_HEAD(dev_list);

        rdma_for_each_port (device, p) {
                if (!rdma_protocol_ib(device, p))
                        continue;
                dev = ipoib_add_port("ib%d", device, p);
                if (!IS_ERR(dev)) {
                        priv = ipoib_priv(dev);
                        list_add_tail(&priv->list, dev_list);
                        count++;
                }
        }

        if (!count) {
                kfree(dev_list);
                return -EOPNOTSUPP;
        }

        ib_set_client_data(device, &ipoib_client, dev_list);
        return 0;
}

static void ipoib_remove_one(struct ib_device *device, void *client_data)
{
        struct ipoib_dev_priv *priv, *tmp, *cpriv, *tcpriv;
        struct list_head *dev_list = client_data;

        list_for_each_entry_safe(priv, tmp, dev_list, list) {
                LIST_HEAD(head);
                ipoib_parent_unregister_pre(priv->dev);

                rtnl_lock();

                netdev_lock(priv->dev);
                list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs,
                                         list)
                        unregister_netdevice_queue(cpriv->dev, &head);
                netdev_unlock(priv->dev);
                unregister_netdevice_queue(priv->dev, &head);
                unregister_netdevice_many(&head);

                rtnl_unlock();
        }

        kfree(dev_list);
}

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
static struct notifier_block ipoib_netdev_notifier = {
        .notifier_call = ipoib_netdev_event,
};
#endif

static int __init ipoib_init_module(void)
{
        int ret;

        ipoib_recvq_size = roundup_pow_of_two(ipoib_recvq_size);
        ipoib_recvq_size = min(ipoib_recvq_size, IPOIB_MAX_QUEUE_SIZE);
        ipoib_recvq_size = max(ipoib_recvq_size, IPOIB_MIN_QUEUE_SIZE);

        ipoib_sendq_size = roundup_pow_of_two(ipoib_sendq_size);
        ipoib_sendq_size = min(ipoib_sendq_size, IPOIB_MAX_QUEUE_SIZE);
        ipoib_sendq_size = max3(ipoib_sendq_size, 2 * MAX_SEND_CQE, IPOIB_MIN_QUEUE_SIZE);
#ifdef CONFIG_INFINIBAND_IPOIB_CM
        ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP);
        ipoib_max_conn_qp = max(ipoib_max_conn_qp, 0);
#endif

        /*
         * When copying small received packets, we only copy from the
         * linear data part of the SKB, so we rely on this condition.
         */
        BUILD_BUG_ON(IPOIB_CM_COPYBREAK > IPOIB_CM_HEAD_SIZE);

        ipoib_register_debugfs();

        /*
         * We create a global workqueue here that is used for all flush
         * operations.  However, if you attempt to flush a workqueue
         * from a task on that same workqueue, it deadlocks the system.
         * We want to be able to flush the tasks associated with a
         * specific net device, so we also create a workqueue for each
         * netdevice.  We queue up the tasks for that device only on
         * its private workqueue, and we only queue up flush events
         * on our global flush workqueue.  This avoids the deadlocks.
         */
        ipoib_workqueue = alloc_ordered_workqueue("ipoib_flush", 0);
        if (!ipoib_workqueue) {
                ret = -ENOMEM;
                goto err_fs;
        }

        ib_sa_register_client(&ipoib_sa_client);

        ret = ib_register_client(&ipoib_client);
        if (ret)
                goto err_sa;

        ret = ipoib_netlink_init();
        if (ret)
                goto err_client;

#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
        register_netdevice_notifier(&ipoib_netdev_notifier);
#endif
        return 0;

err_client:
        ib_unregister_client(&ipoib_client);

err_sa:
        ib_sa_unregister_client(&ipoib_sa_client);
        destroy_workqueue(ipoib_workqueue);

err_fs:
        ipoib_unregister_debugfs();

        return ret;
}

static void __exit ipoib_cleanup_module(void)
{
#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
        unregister_netdevice_notifier(&ipoib_netdev_notifier);
#endif
        ipoib_netlink_fini();
        ib_unregister_client(&ipoib_client);
        ib_sa_unregister_client(&ipoib_sa_client);
        ipoib_unregister_debugfs();
        destroy_workqueue(ipoib_workqueue);
}

module_init(ipoib_init_module);
module_exit(ipoib_cleanup_module);





























































































































   63 
   62 




  201 
  202 











  727 



  728 
  445 










   51 

   46 

  705 

  724 

  727 











  445 




























   20 






   47 





















   12 











  132 


  131 
  131 



  132 



  132 




  563 



  563 











  562 


  561 








   10 




  554 
    2 

















  265 







  264 
  172 





  265 



  263 


































  201 




  202 

  202 

  202 

  171 









   30 

    2 








































  202 














  200 


























  199 
    2 






  186 




   69 







   68 








   69 

   63 

   36 

   28 


    7 



    1 



    1 



   28 


























   63 














   25 
   15 
    3 



   25 




   23 





    5 
   18 






  173 



  172 

    6 
  168 
  173 








    1 


    1 

    1 
    1 







   14 








   14 



   13 

    1 










    8 








    1 

    1 








    1 





    5 

    5 



    5 


    3 
    2 



    2 
    1 





    3 






























































































































    2 

   25 


   24 
    3 





   13 
   47 









    1 

    1 



    1 




    1 



   25 



    9 
    4 























    4 



   27 

    1 


   27 










    2 
   25 


   27 
   27 




   25 
    2 


    2 

   25 

   25 
    2 

   25 
    2 












   27 
   25 
    3 









   38 








   38 



    3 




   27 

    5 



   27 




   27 
   14 

   12 
    2 














    1 





    2 






    2 




















    6 



   26 

    1 

    4 

    4 








   73 
















   68 
    5 



   65 















    8 










   46 











   24 


   55 
    4 

   59 
    5 
   52 






    7 
    7 

    7 










   18 
   59 
    4 
   45 





   13 


   14 


    3 




    3 




    1 




    1 



    8 










    7 





   25 




    3 






    3 

    9 


    2 

   11 







   22 

    9 

   13 


    2 


   23 
   23 




    1 
    3 







    2 


    8 













    1 


    1 



    7 







    4 
    2 



    1 












   14 
   14 







    9 


   14 
    5 



    4 








    9 



    4 








   19 


   19 


   18 

   19 






  285 







  284 
  285 
  285 



  284 


  284 


   21 

  255 




   21 


  274 









   19 











   10 



   19 
    7 



   19 



  283 





















































































   48 





   13 

























   64 




   43 


    3 
    4 



    3 

    4 























 1356 

 1353 

 1357 


  792 


   10 
  545 
   30 




    7 
    3 




  834 







  208 









    8 









  280 







  335 


   43 


   43 


   99 




  265 





   64 

   64 































  291 












  291 






    4 

















  290 




  289 










    3 









    2 
  288 

  291 
  285 
   35 
    2 
  288 
  289 
    4 
    5 
  288 
   14 




  291 



   12 




































































    4 

    4 





    1 


    4 

















   14 





   13 


   14 













   18 


   14 

    4 























   18 


















   18 
   18 






















   18 
   18 
   13 



   18 



   18 

   18 




   14 




    4 





  278 


  278 




















  277 







 1336 
 1339 









 1434 
 1431 



  348 



   33 





















































   17 



   17 




    2 


   14 
    1 












   12 
   12 









   12 


















  816 

  815 

  816 

  816 

  817 









  831 


















  327 


  634 



  634 



  634 



  635 



  633 



  635 










   12 





  816 



  817 











  814 























































   13 










   13 


    2 





    1 


    1 




    3 
    5 
    1 

    4 


















    6 

    2 
    4 
    7 





   12 










   12 














   12 
   12 
   12 
    9 







   12 











    7 










    7 


   12 


































































































































































































































































  616 








  617 











  616 

















  329 








  625 


   10 







  616 








  329 


















   30 

































   30 








































   30 


   30 


















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        NET3        IP device support routines.
 *
 *        Derived from the IP parts of dev.c 1.0.19
 *                 Authors:        Ross Biro
 *                                Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *                                Mark Evans, <evansmp@uhura.aston.ac.uk>
 *
 *        Additional Authors:
 *                Alan Cox, <gw4pts@gw4pts.ampr.org>
 *                Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 *        Changes:
 *                Alexey Kuznetsov:        pa_* fields are replaced with ifaddr
 *                                        lists.
 *                Cyrus Durgin:                updated for kmod
 *                Matthias Andree:        in devinet_ioctl, compare label and
 *                                        address (4.4BSD alias style support),
 *                                        fall back to comparing just the label
 *                                        if no match found.
 */


#include <linux/uaccess.h>
#include <linux/bitops.h>
#include <linux/capability.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/in.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/if_addr.h>
#include <linux/if_ether.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/notifier.h>
#include <linux/inetdevice.h>
#include <linux/igmp.h>
#include "igmp_internal.h"
#include <linux/slab.h>
#include <linux/hash.h>
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif
#include <linux/kmod.h>
#include <linux/netconf.h>

#include <net/arp.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/ip_fib.h>
#include <net/rtnetlink.h>
#include <net/net_namespace.h>
#include <net/addrconf.h>

#define IPV6ONLY_FLAGS        \
                (IFA_F_NODAD | IFA_F_OPTIMISTIC | IFA_F_DADFAILED | \
                 IFA_F_HOMEADDRESS | IFA_F_TENTATIVE | \
                 IFA_F_MANAGETEMPADDR | IFA_F_STABLE_PRIVACY)

static struct ipv4_devconf ipv4_devconf = {
        .data = {
                [IPV4_DEVCONF_ACCEPT_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SEND_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SECURE_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SHARED_MEDIA - 1] = 1,
                [IPV4_DEVCONF_IGMPV2_UNSOLICITED_REPORT_INTERVAL - 1] = 10000 /*ms*/,
                [IPV4_DEVCONF_IGMPV3_UNSOLICITED_REPORT_INTERVAL - 1] =  1000 /*ms*/,
                [IPV4_DEVCONF_ARP_EVICT_NOCARRIER - 1] = 1,
        },
};

static struct ipv4_devconf ipv4_devconf_dflt = {
        .data = {
                [IPV4_DEVCONF_ACCEPT_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SEND_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SECURE_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SHARED_MEDIA - 1] = 1,
                [IPV4_DEVCONF_ACCEPT_SOURCE_ROUTE - 1] = 1,
                [IPV4_DEVCONF_IGMPV2_UNSOLICITED_REPORT_INTERVAL - 1] = 10000 /*ms*/,
                [IPV4_DEVCONF_IGMPV3_UNSOLICITED_REPORT_INTERVAL - 1] =  1000 /*ms*/,
                [IPV4_DEVCONF_ARP_EVICT_NOCARRIER - 1] = 1,
        },
};

#define IPV4_DEVCONF_DFLT(net, attr) \
        IPV4_DEVCONF((*net->ipv4.devconf_dflt), attr)

static const struct nla_policy ifa_ipv4_policy[IFA_MAX+1] = {
        [IFA_LOCAL]             = { .type = NLA_U32 },
        [IFA_ADDRESS]           = { .type = NLA_U32 },
        [IFA_BROADCAST]         = { .type = NLA_U32 },
        [IFA_LABEL]             = { .type = NLA_STRING, .len = IFNAMSIZ - 1 },
        [IFA_CACHEINFO]                = { .len = sizeof(struct ifa_cacheinfo) },
        [IFA_FLAGS]                = { .type = NLA_U32 },
        [IFA_RT_PRIORITY]        = { .type = NLA_U32 },
        [IFA_TARGET_NETNSID]        = { .type = NLA_S32 },
        [IFA_PROTO]                = { .type = NLA_U8 },
};

#define IN4_ADDR_HSIZE_SHIFT        8
#define IN4_ADDR_HSIZE                (1U << IN4_ADDR_HSIZE_SHIFT)

static u32 inet_addr_hash(const struct net *net, __be32 addr)
{
        u32 val = __ipv4_addr_hash(addr, net_hash_mix(net));

        return hash_32(val, IN4_ADDR_HSIZE_SHIFT);
}

static void inet_hash_insert(struct net *net, struct in_ifaddr *ifa)
{
        u32 hash = inet_addr_hash(net, ifa->ifa_local);

        ASSERT_RTNL();
        hlist_add_head_rcu(&ifa->addr_lst, &net->ipv4.inet_addr_lst[hash]);
}

static void inet_hash_remove(struct in_ifaddr *ifa)
{
        ASSERT_RTNL();
        hlist_del_init_rcu(&ifa->addr_lst);
}

/**
 * __ip_dev_find - find the first device with a given source address.
 * @net: the net namespace
 * @addr: the source address
 * @devref: if true, take a reference on the found device
 *
 * If a caller uses devref=false, it should be protected by RCU, or RTNL
 */
struct net_device *__ip_dev_find(struct net *net, __be32 addr, bool devref)
{
        struct net_device *result = NULL;
        struct in_ifaddr *ifa;

        rcu_read_lock();
        ifa = inet_lookup_ifaddr_rcu(net, addr);
        if (!ifa) {
                struct flowi4 fl4 = { .daddr = addr };
                struct fib_result res = { 0 };
                struct fib_table *local;

                /* Fallback to FIB local table so that communication
                 * over loopback subnets work.
                 */
                local = fib_get_table(net, RT_TABLE_LOCAL);
                if (local &&
                    !fib_table_lookup(local, &fl4, &res, FIB_LOOKUP_NOREF) &&
                    res.type == RTN_LOCAL)
                        result = FIB_RES_DEV(res);
        } else {
                result = ifa->ifa_dev->dev;
        }
        if (result && devref)
                dev_hold(result);
        rcu_read_unlock();
        return result;
}
EXPORT_SYMBOL(__ip_dev_find);

/* called under RCU lock */
struct in_ifaddr *inet_lookup_ifaddr_rcu(struct net *net, __be32 addr)
{
        u32 hash = inet_addr_hash(net, addr);
        struct in_ifaddr *ifa;

        hlist_for_each_entry_rcu(ifa, &net->ipv4.inet_addr_lst[hash], addr_lst)
                if (ifa->ifa_local == addr)
                        return ifa;

        return NULL;
}

static void rtmsg_ifa(int event, struct in_ifaddr *, struct nlmsghdr *, u32);

static BLOCKING_NOTIFIER_HEAD(inetaddr_chain);
static BLOCKING_NOTIFIER_HEAD(inetaddr_validator_chain);
static void inet_del_ifa(struct in_device *in_dev,
                         struct in_ifaddr __rcu **ifap,
                         int destroy);
#ifdef CONFIG_SYSCTL
static int devinet_sysctl_register(struct in_device *idev);
static void devinet_sysctl_unregister(struct in_device *idev);
#else
static int devinet_sysctl_register(struct in_device *idev)
{
        return 0;
}
static void devinet_sysctl_unregister(struct in_device *idev)
{
}
#endif

/* Locks all the inet devices. */

static struct in_ifaddr *inet_alloc_ifa(struct in_device *in_dev)
{
        struct in_ifaddr *ifa;

        ifa = kzalloc_obj(*ifa, GFP_KERNEL_ACCOUNT);
        if (!ifa)
                return NULL;

        in_dev_hold(in_dev);
        ifa->ifa_dev = in_dev;

        INIT_HLIST_NODE(&ifa->addr_lst);

        return ifa;
}

static void inet_rcu_free_ifa(struct rcu_head *head)
{
        struct in_ifaddr *ifa = container_of(head, struct in_ifaddr, rcu_head);

        in_dev_put(ifa->ifa_dev);
        kfree(ifa);
}

static void inet_free_ifa(struct in_ifaddr *ifa)
{
        /* Our reference to ifa->ifa_dev must be freed ASAP
         * to release the reference to the netdev the same way.
         * in_dev_put() -> in_dev_finish_destroy() -> netdev_put()
         */
        call_rcu_hurry(&ifa->rcu_head, inet_rcu_free_ifa);
}

static void in_dev_free_rcu(struct rcu_head *head)
{
        struct in_device *idev = container_of(head, struct in_device, rcu_head);

        kfree(rcu_dereference_protected(idev->mc_hash, 1));
        kfree(idev);
}

void in_dev_finish_destroy(struct in_device *idev)
{
        struct net_device *dev = idev->dev;

        WARN_ON(idev->ifa_list);
        WARN_ON(idev->mc_list);
#ifdef NET_REFCNT_DEBUG
        pr_debug("%s: %p=%s\n", __func__, idev, dev ? dev->name : "NIL");
#endif
        netdev_put(dev, &idev->dev_tracker);
        if (!idev->dead)
                pr_err("Freeing alive in_device %p\n", idev);
        else
                call_rcu(&idev->rcu_head, in_dev_free_rcu);
}
EXPORT_SYMBOL(in_dev_finish_destroy);

static struct in_device *inetdev_init(struct net_device *dev)
{
        struct in_device *in_dev;
        int err = -ENOMEM;

        ASSERT_RTNL();

        in_dev = kzalloc_obj(*in_dev);
        if (!in_dev)
                goto out;
        memcpy(&in_dev->cnf, dev_net(dev)->ipv4.devconf_dflt,
                        sizeof(in_dev->cnf));
        in_dev->cnf.sysctl = NULL;
        in_dev->dev = dev;
        in_dev->arp_parms = neigh_parms_alloc(dev, &arp_tbl);
        if (!in_dev->arp_parms)
                goto out_kfree;
        if (IPV4_DEVCONF(in_dev->cnf, FORWARDING))
                netif_disable_lro(dev);
        /* Reference in_dev->dev */
        netdev_hold(dev, &in_dev->dev_tracker, GFP_KERNEL);
        /* Account for reference dev->ip_ptr (below) */
        refcount_set(&in_dev->refcnt, 1);

        if (dev != blackhole_netdev) {
                err = devinet_sysctl_register(in_dev);
                if (err) {
                        in_dev->dead = 1;
                        neigh_parms_release(&arp_tbl, in_dev->arp_parms);
                        in_dev_put(in_dev);
                        in_dev = NULL;
                        goto out;
                }
                ip_mc_init_dev(in_dev);
                if (dev->flags & IFF_UP)
                        ip_mc_up(in_dev);
        }

        /* we can receive as soon as ip_ptr is set -- do this last */
        rcu_assign_pointer(dev->ip_ptr, in_dev);
out:
        return in_dev ?: ERR_PTR(err);
out_kfree:
        kfree(in_dev);
        in_dev = NULL;
        goto out;
}

static void inetdev_destroy(struct in_device *in_dev)
{
        struct net_device *dev;
        struct in_ifaddr *ifa;

        ASSERT_RTNL();

        dev = in_dev->dev;

        in_dev->dead = 1;

        ip_mc_destroy_dev(in_dev);

        while ((ifa = rtnl_dereference(in_dev->ifa_list)) != NULL) {
                inet_del_ifa(in_dev, &in_dev->ifa_list, 0);
                inet_free_ifa(ifa);
        }

        RCU_INIT_POINTER(dev->ip_ptr, NULL);

        devinet_sysctl_unregister(in_dev);
        neigh_parms_release(&arp_tbl, in_dev->arp_parms);
        arp_ifdown(dev);

        in_dev_put(in_dev);
}

static int __init inet_blackhole_dev_init(void)
{
        struct in_device *in_dev;

        rtnl_lock();
        in_dev = inetdev_init(blackhole_netdev);
        rtnl_unlock();

        return PTR_ERR_OR_ZERO(in_dev);
}
late_initcall(inet_blackhole_dev_init);

int inet_addr_onlink(struct in_device *in_dev, __be32 a, __be32 b)
{
        const struct in_ifaddr *ifa;

        rcu_read_lock();
        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                if (inet_ifa_match(a, ifa)) {
                        if (!b || inet_ifa_match(b, ifa)) {
                                rcu_read_unlock();
                                return 1;
                        }
                }
        }
        rcu_read_unlock();
        return 0;
}

static void __inet_del_ifa(struct in_device *in_dev,
                           struct in_ifaddr __rcu **ifap,
                           int destroy, struct nlmsghdr *nlh, u32 portid)
{
        struct in_ifaddr *promote = NULL;
        struct in_ifaddr *ifa, *ifa1;
        struct in_ifaddr __rcu **last_prim;
        struct in_ifaddr *prev_prom = NULL;
        int do_promote = IN_DEV_PROMOTE_SECONDARIES(in_dev);

        ASSERT_RTNL();

        ifa1 = rtnl_dereference(*ifap);
        last_prim = ifap;
        if (in_dev->dead)
                goto no_promotions;

        /* 1. Deleting primary ifaddr forces deletion all secondaries
         * unless alias promotion is set
         **/

        if (!(ifa1->ifa_flags & IFA_F_SECONDARY)) {
                struct in_ifaddr __rcu **ifap1 = &ifa1->ifa_next;

                while ((ifa = rtnl_dereference(*ifap1)) != NULL) {
                        if (!(ifa->ifa_flags & IFA_F_SECONDARY) &&
                            ifa1->ifa_scope <= ifa->ifa_scope)
                                last_prim = &ifa->ifa_next;

                        if (!(ifa->ifa_flags & IFA_F_SECONDARY) ||
                            ifa1->ifa_mask != ifa->ifa_mask ||
                            !inet_ifa_match(ifa1->ifa_address, ifa)) {
                                ifap1 = &ifa->ifa_next;
                                prev_prom = ifa;
                                continue;
                        }

                        if (!do_promote) {
                                inet_hash_remove(ifa);
                                *ifap1 = ifa->ifa_next;

                                rtmsg_ifa(RTM_DELADDR, ifa, nlh, portid);
                                blocking_notifier_call_chain(&inetaddr_chain,
                                                NETDEV_DOWN, ifa);
                                inet_free_ifa(ifa);
                        } else {
                                promote = ifa;
                                break;
                        }
                }
        }

        /* On promotion all secondaries from subnet are changing
         * the primary IP, we must remove all their routes silently
         * and later to add them back with new prefsrc. Do this
         * while all addresses are on the device list.
         */
        for (ifa = promote; ifa; ifa = rtnl_dereference(ifa->ifa_next)) {
                if (ifa1->ifa_mask == ifa->ifa_mask &&
                    inet_ifa_match(ifa1->ifa_address, ifa))
                        fib_del_ifaddr(ifa, ifa1);
        }

no_promotions:
        /* 2. Unlink it */

        *ifap = ifa1->ifa_next;
        inet_hash_remove(ifa1);

        /* 3. Announce address deletion */

        /* Send message first, then call notifier.
           At first sight, FIB update triggered by notifier
           will refer to already deleted ifaddr, that could confuse
           netlink listeners. It is not true: look, gated sees
           that route deleted and if it still thinks that ifaddr
           is valid, it will try to restore deleted routes... Grr.
           So that, this order is correct.
         */
        rtmsg_ifa(RTM_DELADDR, ifa1, nlh, portid);
        blocking_notifier_call_chain(&inetaddr_chain, NETDEV_DOWN, ifa1);

        if (promote) {
                struct in_ifaddr *next_sec;

                next_sec = rtnl_dereference(promote->ifa_next);
                if (prev_prom) {
                        struct in_ifaddr *last_sec;

                        rcu_assign_pointer(prev_prom->ifa_next, next_sec);

                        last_sec = rtnl_dereference(*last_prim);
                        rcu_assign_pointer(promote->ifa_next, last_sec);
                        rcu_assign_pointer(*last_prim, promote);
                }

                promote->ifa_flags &= ~IFA_F_SECONDARY;
                rtmsg_ifa(RTM_NEWADDR, promote, nlh, portid);
                blocking_notifier_call_chain(&inetaddr_chain,
                                NETDEV_UP, promote);
                for (ifa = next_sec; ifa;
                     ifa = rtnl_dereference(ifa->ifa_next)) {
                        if (ifa1->ifa_mask != ifa->ifa_mask ||
                            !inet_ifa_match(ifa1->ifa_address, ifa))
                                        continue;
                        fib_add_ifaddr(ifa);
                }

        }
        if (destroy)
                inet_free_ifa(ifa1);
}

static void inet_del_ifa(struct in_device *in_dev,
                         struct in_ifaddr __rcu **ifap,
                         int destroy)
{
        __inet_del_ifa(in_dev, ifap, destroy, NULL, 0);
}

static int __inet_insert_ifa(struct in_ifaddr *ifa, struct nlmsghdr *nlh,
                             u32 portid, struct netlink_ext_ack *extack)
{
        struct in_ifaddr __rcu **last_primary, **ifap;
        struct in_device *in_dev = ifa->ifa_dev;
        struct net *net = dev_net(in_dev->dev);
        struct in_validator_info ivi;
        struct in_ifaddr *ifa1;
        int ret;

        ASSERT_RTNL();

        ifa->ifa_flags &= ~IFA_F_SECONDARY;
        last_primary = &in_dev->ifa_list;

        /* Don't set IPv6 only flags to IPv4 addresses */
        ifa->ifa_flags &= ~IPV6ONLY_FLAGS;

        ifap = &in_dev->ifa_list;
        ifa1 = rtnl_dereference(*ifap);

        while (ifa1) {
                if (!(ifa1->ifa_flags & IFA_F_SECONDARY) &&
                    ifa->ifa_scope <= ifa1->ifa_scope)
                        last_primary = &ifa1->ifa_next;
                if (ifa1->ifa_mask == ifa->ifa_mask &&
                    inet_ifa_match(ifa1->ifa_address, ifa)) {
                        if (ifa1->ifa_local == ifa->ifa_local) {
                                inet_free_ifa(ifa);
                                return -EEXIST;
                        }
                        if (ifa1->ifa_scope != ifa->ifa_scope) {
                                NL_SET_ERR_MSG(extack, "ipv4: Invalid scope value");
                                inet_free_ifa(ifa);
                                return -EINVAL;
                        }
                        ifa->ifa_flags |= IFA_F_SECONDARY;
                }

                ifap = &ifa1->ifa_next;
                ifa1 = rtnl_dereference(*ifap);
        }

        /* Allow any devices that wish to register ifaddr validtors to weigh
         * in now, before changes are committed.  The rntl lock is serializing
         * access here, so the state should not change between a validator call
         * and a final notify on commit.  This isn't invoked on promotion under
         * the assumption that validators are checking the address itself, and
         * not the flags.
         */
        ivi.ivi_addr = ifa->ifa_address;
        ivi.ivi_dev = ifa->ifa_dev;
        ivi.extack = extack;
        ret = blocking_notifier_call_chain(&inetaddr_validator_chain,
                                           NETDEV_UP, &ivi);
        ret = notifier_to_errno(ret);
        if (ret) {
                inet_free_ifa(ifa);
                return ret;
        }

        if (!(ifa->ifa_flags & IFA_F_SECONDARY))
                ifap = last_primary;

        rcu_assign_pointer(ifa->ifa_next, *ifap);
        rcu_assign_pointer(*ifap, ifa);

        inet_hash_insert(dev_net(in_dev->dev), ifa);

        cancel_delayed_work(&net->ipv4.addr_chk_work);
        queue_delayed_work(system_power_efficient_wq, &net->ipv4.addr_chk_work, 0);

        /* Send message first, then call notifier.
           Notifier will trigger FIB update, so that
           listeners of netlink will know about new ifaddr */
        rtmsg_ifa(RTM_NEWADDR, ifa, nlh, portid);
        blocking_notifier_call_chain(&inetaddr_chain, NETDEV_UP, ifa);

        return 0;
}

static int inet_insert_ifa(struct in_ifaddr *ifa)
{
        if (!ifa->ifa_local) {
                inet_free_ifa(ifa);
                return 0;
        }

        return __inet_insert_ifa(ifa, NULL, 0, NULL);
}

static int inet_set_ifa(struct net_device *dev, struct in_ifaddr *ifa)
{
        struct in_device *in_dev = __in_dev_get_rtnl_net(dev);

        ipv4_devconf_setall(in_dev);
        neigh_parms_data_state_setall(in_dev->arp_parms);

        if (ipv4_is_loopback(ifa->ifa_local))
                ifa->ifa_scope = RT_SCOPE_HOST;
        return inet_insert_ifa(ifa);
}

/* Caller must hold RCU or RTNL :
 * We dont take a reference on found in_device
 */
struct in_device *inetdev_by_index(struct net *net, int ifindex)
{
        struct net_device *dev;
        struct in_device *in_dev = NULL;

        rcu_read_lock();
        dev = dev_get_by_index_rcu(net, ifindex);
        if (dev)
                in_dev = rcu_dereference_rtnl(dev->ip_ptr);
        rcu_read_unlock();
        return in_dev;
}
EXPORT_SYMBOL(inetdev_by_index);

/* Called only from RTNL semaphored context. No locks. */

struct in_ifaddr *inet_ifa_byprefix(struct in_device *in_dev, __be32 prefix,
                                    __be32 mask)
{
        struct in_ifaddr *ifa;

        ASSERT_RTNL();

        in_dev_for_each_ifa_rtnl(ifa, in_dev) {
                if (ifa->ifa_mask == mask && inet_ifa_match(prefix, ifa))
                        return ifa;
        }
        return NULL;
}

static int ip_mc_autojoin_config(struct net *net, bool join,
                                 const struct in_ifaddr *ifa)
{
#if defined(CONFIG_IP_MULTICAST)
        struct ip_mreqn mreq = {
                .imr_multiaddr.s_addr = ifa->ifa_address,
                .imr_ifindex = ifa->ifa_dev->dev->ifindex,
        };
        struct sock *sk = net->ipv4.mc_autojoin_sk;
        int ret;

        ASSERT_RTNL_NET(net);

        lock_sock(sk);
        if (join)
                ret = ip_mc_join_group(sk, &mreq);
        else
                ret = ip_mc_leave_group(sk, &mreq);
        release_sock(sk);

        return ret;
#else
        return -EOPNOTSUPP;
#endif
}

static int inet_rtm_deladdr(struct sk_buff *skb, struct nlmsghdr *nlh,
                            struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct in_ifaddr __rcu **ifap;
        struct nlattr *tb[IFA_MAX+1];
        struct in_device *in_dev;
        struct ifaddrmsg *ifm;
        struct in_ifaddr *ifa;
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFA_MAX,
                                     ifa_ipv4_policy, extack);
        if (err < 0)
                goto out;

        ifm = nlmsg_data(nlh);

        rtnl_net_lock(net);

        in_dev = inetdev_by_index(net, ifm->ifa_index);
        if (!in_dev) {
                NL_SET_ERR_MSG(extack, "ipv4: Device not found");
                err = -ENODEV;
                goto unlock;
        }

        for (ifap = &in_dev->ifa_list;
             (ifa = rtnl_net_dereference(net, *ifap)) != NULL;
             ifap = &ifa->ifa_next) {
                if (tb[IFA_LOCAL] &&
                    ifa->ifa_local != nla_get_in_addr(tb[IFA_LOCAL]))
                        continue;

                if (tb[IFA_LABEL] && nla_strcmp(tb[IFA_LABEL], ifa->ifa_label))
                        continue;

                if (tb[IFA_ADDRESS] &&
                    (ifm->ifa_prefixlen != ifa->ifa_prefixlen ||
                    !inet_ifa_match(nla_get_in_addr(tb[IFA_ADDRESS]), ifa)))
                        continue;

                if (ipv4_is_multicast(ifa->ifa_address))
                        ip_mc_autojoin_config(net, false, ifa);

                __inet_del_ifa(in_dev, ifap, 1, nlh, NETLINK_CB(skb).portid);
                goto unlock;
        }

        NL_SET_ERR_MSG(extack, "ipv4: Address not found");
        err = -EADDRNOTAVAIL;
unlock:
        rtnl_net_unlock(net);
out:
        return err;
}

static void check_lifetime(struct work_struct *work)
{
        unsigned long now, next, next_sec, next_sched;
        struct in_ifaddr *ifa;
        struct hlist_node *n;
        struct net *net;
        int i;

        net = container_of(to_delayed_work(work), struct net, ipv4.addr_chk_work);
        now = jiffies;
        next = round_jiffies_up(now + ADDR_CHECK_FREQUENCY);

        for (i = 0; i < IN4_ADDR_HSIZE; i++) {
                struct hlist_head *head = &net->ipv4.inet_addr_lst[i];
                bool change_needed = false;

                rcu_read_lock();
                hlist_for_each_entry_rcu(ifa, head, addr_lst) {
                        unsigned long age, tstamp;
                        u32 preferred_lft;
                        u32 valid_lft;
                        u32 flags;

                        flags = READ_ONCE(ifa->ifa_flags);
                        if (flags & IFA_F_PERMANENT)
                                continue;

                        preferred_lft = READ_ONCE(ifa->ifa_preferred_lft);
                        valid_lft = READ_ONCE(ifa->ifa_valid_lft);
                        tstamp = READ_ONCE(ifa->ifa_tstamp);
                        /* We try to batch several events at once. */
                        age = (now - tstamp +
                               ADDRCONF_TIMER_FUZZ_MINUS) / HZ;

                        if (valid_lft != INFINITY_LIFE_TIME &&
                            age >= valid_lft) {
                                change_needed = true;
                        } else if (preferred_lft ==
                                   INFINITY_LIFE_TIME) {
                                continue;
                        } else if (age >= preferred_lft) {
                                if (time_before(tstamp + valid_lft * HZ, next))
                                        next = tstamp + valid_lft * HZ;

                                if (!(flags & IFA_F_DEPRECATED))
                                        change_needed = true;
                        } else if (time_before(tstamp + preferred_lft * HZ,
                                               next)) {
                                next = tstamp + preferred_lft * HZ;
                        }
                }
                rcu_read_unlock();
                if (!change_needed)
                        continue;

                rtnl_net_lock(net);
                hlist_for_each_entry_safe(ifa, n, head, addr_lst) {
                        unsigned long age;

                        if (ifa->ifa_flags & IFA_F_PERMANENT)
                                continue;

                        /* We try to batch several events at once. */
                        age = (now - ifa->ifa_tstamp +
                               ADDRCONF_TIMER_FUZZ_MINUS) / HZ;

                        if (ifa->ifa_valid_lft != INFINITY_LIFE_TIME &&
                            age >= ifa->ifa_valid_lft) {
                                struct in_ifaddr __rcu **ifap;
                                struct in_ifaddr *tmp;

                                ifap = &ifa->ifa_dev->ifa_list;
                                tmp = rtnl_net_dereference(net, *ifap);
                                while (tmp) {
                                        if (tmp == ifa) {
                                                inet_del_ifa(ifa->ifa_dev,
                                                             ifap, 1);
                                                break;
                                        }
                                        ifap = &tmp->ifa_next;
                                        tmp = rtnl_net_dereference(net, *ifap);
                                }
                        } else if (ifa->ifa_preferred_lft !=
                                   INFINITY_LIFE_TIME &&
                                   age >= ifa->ifa_preferred_lft &&
                                   !(ifa->ifa_flags & IFA_F_DEPRECATED)) {
                                ifa->ifa_flags |= IFA_F_DEPRECATED;
                                rtmsg_ifa(RTM_NEWADDR, ifa, NULL, 0);
                        }
                }
                rtnl_net_unlock(net);
        }

        next_sec = round_jiffies_up(next);
        next_sched = next;

        /* If rounded timeout is accurate enough, accept it. */
        if (time_before(next_sec, next + ADDRCONF_TIMER_FUZZ))
                next_sched = next_sec;

        now = jiffies;
        /* And minimum interval is ADDRCONF_TIMER_FUZZ_MAX. */
        if (time_before(next_sched, now + ADDRCONF_TIMER_FUZZ_MAX))
                next_sched = now + ADDRCONF_TIMER_FUZZ_MAX;

        queue_delayed_work(system_power_efficient_wq, &net->ipv4.addr_chk_work,
                           next_sched - now);
}

static void set_ifa_lifetime(struct in_ifaddr *ifa, __u32 valid_lft,
                             __u32 prefered_lft)
{
        unsigned long timeout;
        u32 flags;

        flags = ifa->ifa_flags & ~(IFA_F_PERMANENT | IFA_F_DEPRECATED);

        timeout = addrconf_timeout_fixup(valid_lft, HZ);
        if (addrconf_finite_timeout(timeout))
                WRITE_ONCE(ifa->ifa_valid_lft, timeout);
        else
                flags |= IFA_F_PERMANENT;

        timeout = addrconf_timeout_fixup(prefered_lft, HZ);
        if (addrconf_finite_timeout(timeout)) {
                if (timeout == 0)
                        flags |= IFA_F_DEPRECATED;
                WRITE_ONCE(ifa->ifa_preferred_lft, timeout);
        }
        WRITE_ONCE(ifa->ifa_flags, flags);
        WRITE_ONCE(ifa->ifa_tstamp, jiffies);
        if (!ifa->ifa_cstamp)
                WRITE_ONCE(ifa->ifa_cstamp, ifa->ifa_tstamp);
}

static int inet_validate_rtm(struct nlmsghdr *nlh, struct nlattr **tb,
                             struct netlink_ext_ack *extack,
                             __u32 *valid_lft, __u32 *prefered_lft)
{
        struct ifaddrmsg *ifm = nlmsg_data(nlh);
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFA_MAX,
                                     ifa_ipv4_policy, extack);
        if (err < 0)
                return err;

        if (ifm->ifa_prefixlen > 32) {
                NL_SET_ERR_MSG(extack, "ipv4: Invalid prefix length");
                return -EINVAL;
        }

        if (!tb[IFA_LOCAL]) {
                NL_SET_ERR_MSG(extack, "ipv4: Local address is not supplied");
                return -EINVAL;
        }

        if (tb[IFA_CACHEINFO]) {
                struct ifa_cacheinfo *ci;

                ci = nla_data(tb[IFA_CACHEINFO]);
                if (!ci->ifa_valid || ci->ifa_prefered > ci->ifa_valid) {
                        NL_SET_ERR_MSG(extack, "ipv4: address lifetime invalid");
                        return -EINVAL;
                }

                *valid_lft = ci->ifa_valid;
                *prefered_lft = ci->ifa_prefered;
        }

        return 0;
}

static struct in_ifaddr *inet_rtm_to_ifa(struct net *net, struct nlmsghdr *nlh,
                                         struct nlattr **tb,
                                         struct netlink_ext_ack *extack)
{
        struct ifaddrmsg *ifm = nlmsg_data(nlh);
        struct in_device *in_dev;
        struct net_device *dev;
        struct in_ifaddr *ifa;
        int err;

        dev = __dev_get_by_index(net, ifm->ifa_index);
        err = -ENODEV;
        if (!dev) {
                NL_SET_ERR_MSG(extack, "ipv4: Device not found");
                goto errout;
        }

        in_dev = __in_dev_get_rtnl_net(dev);
        err = -ENOBUFS;
        if (!in_dev)
                goto errout;

        ifa = inet_alloc_ifa(in_dev);
        if (!ifa)
                /*
                 * A potential indev allocation can be left alive, it stays
                 * assigned to its device and is destroy with it.
                 */
                goto errout;

        ipv4_devconf_setall(in_dev);
        neigh_parms_data_state_setall(in_dev->arp_parms);

        if (!tb[IFA_ADDRESS])
                tb[IFA_ADDRESS] = tb[IFA_LOCAL];

        ifa->ifa_prefixlen = ifm->ifa_prefixlen;
        ifa->ifa_mask = inet_make_mask(ifm->ifa_prefixlen);
        ifa->ifa_flags = nla_get_u32_default(tb[IFA_FLAGS], ifm->ifa_flags);
        ifa->ifa_scope = ifm->ifa_scope;
        ifa->ifa_local = nla_get_in_addr(tb[IFA_LOCAL]);
        ifa->ifa_address = nla_get_in_addr(tb[IFA_ADDRESS]);

        if (tb[IFA_BROADCAST])
                ifa->ifa_broadcast = nla_get_in_addr(tb[IFA_BROADCAST]);

        if (tb[IFA_LABEL])
                nla_strscpy(ifa->ifa_label, tb[IFA_LABEL], IFNAMSIZ);
        else
                memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);

        if (tb[IFA_RT_PRIORITY])
                ifa->ifa_rt_priority = nla_get_u32(tb[IFA_RT_PRIORITY]);

        if (tb[IFA_PROTO])
                ifa->ifa_proto = nla_get_u8(tb[IFA_PROTO]);

        return ifa;

errout:
        return ERR_PTR(err);
}

static struct in_ifaddr *find_matching_ifa(struct net *net, struct in_ifaddr *ifa)
{
        struct in_device *in_dev = ifa->ifa_dev;
        struct in_ifaddr *ifa1;

        in_dev_for_each_ifa_rtnl_net(net, ifa1, in_dev) {
                if (ifa1->ifa_mask == ifa->ifa_mask &&
                    inet_ifa_match(ifa1->ifa_address, ifa) &&
                    ifa1->ifa_local == ifa->ifa_local)
                        return ifa1;
        }

        return NULL;
}

static int inet_rtm_newaddr(struct sk_buff *skb, struct nlmsghdr *nlh,
                            struct netlink_ext_ack *extack)
{
        __u32 prefered_lft = INFINITY_LIFE_TIME;
        __u32 valid_lft = INFINITY_LIFE_TIME;
        struct net *net = sock_net(skb->sk);
        struct in_ifaddr *ifa_existing;
        struct nlattr *tb[IFA_MAX + 1];
        struct in_ifaddr *ifa;
        int ret;

        ret = inet_validate_rtm(nlh, tb, extack, &valid_lft, &prefered_lft);
        if (ret < 0)
                return ret;

        if (!nla_get_in_addr(tb[IFA_LOCAL]))
                return 0;

        rtnl_net_lock(net);

        ifa = inet_rtm_to_ifa(net, nlh, tb, extack);
        if (IS_ERR(ifa)) {
                ret = PTR_ERR(ifa);
                goto unlock;
        }

        ifa_existing = find_matching_ifa(net, ifa);
        if (!ifa_existing) {
                /* It would be best to check for !NLM_F_CREATE here but
                 * userspace already relies on not having to provide this.
                 */
                set_ifa_lifetime(ifa, valid_lft, prefered_lft);
                if (ifa->ifa_flags & IFA_F_MCAUTOJOIN) {
                        ret = ip_mc_autojoin_config(net, true, ifa);
                        if (ret < 0) {
                                NL_SET_ERR_MSG(extack, "ipv4: Multicast auto join failed");
                                inet_free_ifa(ifa);
                                goto unlock;
                        }
                }

                ret = __inet_insert_ifa(ifa, nlh, NETLINK_CB(skb).portid, extack);
        } else {
                u32 new_metric = ifa->ifa_rt_priority;
                u8 new_proto = ifa->ifa_proto;

                inet_free_ifa(ifa);

                if (nlh->nlmsg_flags & NLM_F_EXCL ||
                    !(nlh->nlmsg_flags & NLM_F_REPLACE)) {
                        NL_SET_ERR_MSG(extack, "ipv4: Address already assigned");
                        ret = -EEXIST;
                        goto unlock;
                }
                ifa = ifa_existing;

                if (ifa->ifa_rt_priority != new_metric) {
                        fib_modify_prefix_metric(ifa, new_metric);
                        ifa->ifa_rt_priority = new_metric;
                }

                ifa->ifa_proto = new_proto;

                set_ifa_lifetime(ifa, valid_lft, prefered_lft);
                cancel_delayed_work(&net->ipv4.addr_chk_work);
                queue_delayed_work(system_power_efficient_wq,
                                   &net->ipv4.addr_chk_work, 0);
                rtmsg_ifa(RTM_NEWADDR, ifa, nlh, NETLINK_CB(skb).portid);
        }

unlock:
        rtnl_net_unlock(net);

        return ret;
}

/*
 *        Determine a default network mask, based on the IP address.
 */

static int inet_abc_len(__be32 addr)
{
        int rc = -1;        /* Something else, probably a multicast. */

        if (ipv4_is_zeronet(addr) || ipv4_is_lbcast(addr))
                rc = 0;
        else {
                __u32 haddr = ntohl(addr);
                if (IN_CLASSA(haddr))
                        rc = 8;
                else if (IN_CLASSB(haddr))
                        rc = 16;
                else if (IN_CLASSC(haddr))
                        rc = 24;
                else if (IN_CLASSE(haddr))
                        rc = 32;
        }

        return rc;
}


int devinet_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr)
{
        struct sockaddr_in sin_orig;
        struct sockaddr_in *sin = (struct sockaddr_in *)&ifr->ifr_addr;
        struct in_ifaddr __rcu **ifap = NULL;
        struct in_device *in_dev;
        struct in_ifaddr *ifa = NULL;
        struct net_device *dev;
        char *colon;
        int ret = -EFAULT;
        int tryaddrmatch = 0;

        ifr->ifr_name[IFNAMSIZ - 1] = 0;

        /* save original address for comparison */
        memcpy(&sin_orig, sin, sizeof(*sin));

        colon = strchr(ifr->ifr_name, ':');
        if (colon)
                *colon = 0;

        dev_load(net, ifr->ifr_name);

        switch (cmd) {
        case SIOCGIFADDR:        /* Get interface address */
        case SIOCGIFBRDADDR:        /* Get the broadcast address */
        case SIOCGIFDSTADDR:        /* Get the destination address */
        case SIOCGIFNETMASK:        /* Get the netmask for the interface */
                /* Note that these ioctls will not sleep,
                   so that we do not impose a lock.
                   One day we will be forced to put shlock here (I mean SMP)
                 */
                tryaddrmatch = (sin_orig.sin_family == AF_INET);
                memset(sin, 0, sizeof(*sin));
                sin->sin_family = AF_INET;
                break;

        case SIOCSIFFLAGS:
                ret = -EPERM;
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        goto out;
                break;
        case SIOCSIFADDR:        /* Set interface address (and family) */
        case SIOCSIFBRDADDR:        /* Set the broadcast address */
        case SIOCSIFDSTADDR:        /* Set the destination address */
        case SIOCSIFNETMASK:         /* Set the netmask for the interface */
                ret = -EPERM;
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        goto out;
                ret = -EINVAL;
                if (sin->sin_family != AF_INET)
                        goto out;
                break;
        default:
                ret = -EINVAL;
                goto out;
        }

        rtnl_net_lock(net);

        ret = -ENODEV;
        dev = __dev_get_by_name(net, ifr->ifr_name);
        if (!dev)
                goto done;

        if (colon)
                *colon = ':';

        in_dev = __in_dev_get_rtnl_net(dev);
        if (in_dev) {
                if (tryaddrmatch) {
                        /* Matthias Andree */
                        /* compare label and address (4.4BSD style) */
                        /* note: we only do this for a limited set of ioctls
                           and only if the original address family was AF_INET.
                           This is checked above. */

                        for (ifap = &in_dev->ifa_list;
                             (ifa = rtnl_net_dereference(net, *ifap)) != NULL;
                             ifap = &ifa->ifa_next) {
                                if (!strcmp(ifr->ifr_name, ifa->ifa_label) &&
                                    sin_orig.sin_addr.s_addr ==
                                                        ifa->ifa_local) {
                                        break; /* found */
                                }
                        }
                }
                /* we didn't get a match, maybe the application is
                   4.3BSD-style and passed in junk so we fall back to
                   comparing just the label */
                if (!ifa) {
                        for (ifap = &in_dev->ifa_list;
                             (ifa = rtnl_net_dereference(net, *ifap)) != NULL;
                             ifap = &ifa->ifa_next)
                                if (!strcmp(ifr->ifr_name, ifa->ifa_label))
                                        break;
                }
        }

        ret = -EADDRNOTAVAIL;
        if (!ifa && cmd != SIOCSIFADDR && cmd != SIOCSIFFLAGS)
                goto done;

        switch (cmd) {
        case SIOCGIFADDR:        /* Get interface address */
                ret = 0;
                sin->sin_addr.s_addr = ifa->ifa_local;
                break;

        case SIOCGIFBRDADDR:        /* Get the broadcast address */
                ret = 0;
                sin->sin_addr.s_addr = ifa->ifa_broadcast;
                break;

        case SIOCGIFDSTADDR:        /* Get the destination address */
                ret = 0;
                sin->sin_addr.s_addr = ifa->ifa_address;
                break;

        case SIOCGIFNETMASK:        /* Get the netmask for the interface */
                ret = 0;
                sin->sin_addr.s_addr = ifa->ifa_mask;
                break;

        case SIOCSIFFLAGS:
                if (colon) {
                        ret = -EADDRNOTAVAIL;
                        if (!ifa)
                                break;
                        ret = 0;
                        if (!(ifr->ifr_flags & IFF_UP))
                                inet_del_ifa(in_dev, ifap, 1);
                        break;
                }

                /* NETDEV_UP/DOWN/CHANGE could touch a peer dev */
                ASSERT_RTNL();
                ret = dev_change_flags(dev, ifr->ifr_flags, NULL);
                break;

        case SIOCSIFADDR:        /* Set interface address (and family) */
                ret = -EINVAL;
                if (inet_abc_len(sin->sin_addr.s_addr) < 0)
                        break;

                if (!ifa) {
                        ret = -ENOBUFS;
                        if (!in_dev)
                                break;
                        ifa = inet_alloc_ifa(in_dev);
                        if (!ifa)
                                break;

                        if (colon)
                                memcpy(ifa->ifa_label, ifr->ifr_name, IFNAMSIZ);
                        else
                                memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);
                } else {
                        ret = 0;
                        if (ifa->ifa_local == sin->sin_addr.s_addr)
                                break;
                        inet_del_ifa(in_dev, ifap, 0);
                        ifa->ifa_broadcast = 0;
                        ifa->ifa_scope = 0;
                }

                ifa->ifa_address = ifa->ifa_local = sin->sin_addr.s_addr;

                if (!(dev->flags & IFF_POINTOPOINT)) {
                        ifa->ifa_prefixlen = inet_abc_len(ifa->ifa_address);
                        ifa->ifa_mask = inet_make_mask(ifa->ifa_prefixlen);
                        if ((dev->flags & IFF_BROADCAST) &&
                            ifa->ifa_prefixlen < 31)
                                ifa->ifa_broadcast = ifa->ifa_address |
                                                     ~ifa->ifa_mask;
                } else {
                        ifa->ifa_prefixlen = 32;
                        ifa->ifa_mask = inet_make_mask(32);
                }
                set_ifa_lifetime(ifa, INFINITY_LIFE_TIME, INFINITY_LIFE_TIME);
                ret = inet_set_ifa(dev, ifa);
                break;

        case SIOCSIFBRDADDR:        /* Set the broadcast address */
                ret = 0;
                if (ifa->ifa_broadcast != sin->sin_addr.s_addr) {
                        inet_del_ifa(in_dev, ifap, 0);
                        ifa->ifa_broadcast = sin->sin_addr.s_addr;
                        inet_insert_ifa(ifa);
                }
                break;

        case SIOCSIFDSTADDR:        /* Set the destination address */
                ret = 0;
                if (ifa->ifa_address == sin->sin_addr.s_addr)
                        break;
                ret = -EINVAL;
                if (inet_abc_len(sin->sin_addr.s_addr) < 0)
                        break;
                ret = 0;
                inet_del_ifa(in_dev, ifap, 0);
                ifa->ifa_address = sin->sin_addr.s_addr;
                inet_insert_ifa(ifa);
                break;

        case SIOCSIFNETMASK:         /* Set the netmask for the interface */

                /*
                 *        The mask we set must be legal.
                 */
                ret = -EINVAL;
                if (bad_mask(sin->sin_addr.s_addr, 0))
                        break;
                ret = 0;
                if (ifa->ifa_mask != sin->sin_addr.s_addr) {
                        __be32 old_mask = ifa->ifa_mask;
                        inet_del_ifa(in_dev, ifap, 0);
                        ifa->ifa_mask = sin->sin_addr.s_addr;
                        ifa->ifa_prefixlen = inet_mask_len(ifa->ifa_mask);

                        /* See if current broadcast address matches
                         * with current netmask, then recalculate
                         * the broadcast address. Otherwise it's a
                         * funny address, so don't touch it since
                         * the user seems to know what (s)he's doing...
                         */
                        if ((dev->flags & IFF_BROADCAST) &&
                            (ifa->ifa_prefixlen < 31) &&
                            (ifa->ifa_broadcast ==
                             (ifa->ifa_local|~old_mask))) {
                                ifa->ifa_broadcast = (ifa->ifa_local |
                                                      ~sin->sin_addr.s_addr);
                        }
                        inet_insert_ifa(ifa);
                }
                break;
        }
done:
        rtnl_net_unlock(net);
out:
        return ret;
}

int inet_gifconf(struct net_device *dev, char __user *buf, int len, int size)
{
        struct in_device *in_dev = __in_dev_get_rtnl_net(dev);
        const struct in_ifaddr *ifa;
        struct ifreq ifr;
        int done = 0;

        if (WARN_ON(size > sizeof(struct ifreq)))
                goto out;

        if (!in_dev)
                goto out;

        in_dev_for_each_ifa_rtnl_net(dev_net(dev), ifa, in_dev) {
                if (!buf) {
                        done += size;
                        continue;
                }
                if (len < size)
                        break;
                memset(&ifr, 0, sizeof(struct ifreq));
                strcpy(ifr.ifr_name, ifa->ifa_label);

                (*(struct sockaddr_in *)&ifr.ifr_addr).sin_family = AF_INET;
                (*(struct sockaddr_in *)&ifr.ifr_addr).sin_addr.s_addr =
                                                                ifa->ifa_local;

                if (copy_to_user(buf + done, &ifr, size)) {
                        done = -EFAULT;
                        break;
                }
                len  -= size;
                done += size;
        }
out:
        return done;
}

static __be32 in_dev_select_addr(const struct in_device *in_dev,
                                 int scope)
{
        const struct in_ifaddr *ifa;

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                if (READ_ONCE(ifa->ifa_flags) & IFA_F_SECONDARY)
                        continue;
                if (ifa->ifa_scope != RT_SCOPE_LINK &&
                    ifa->ifa_scope <= scope)
                        return ifa->ifa_local;
        }

        return 0;
}

__be32 inet_select_addr(const struct net_device *dev, __be32 dst, int scope)
{
        const struct in_ifaddr *ifa;
        __be32 addr = 0;
        unsigned char localnet_scope = RT_SCOPE_HOST;
        struct in_device *in_dev;
        struct net *net;
        int master_idx;

        rcu_read_lock();
        net = dev_net_rcu(dev);
        in_dev = __in_dev_get_rcu(dev);
        if (!in_dev)
                goto no_in_dev;

        if (unlikely(IN_DEV_ROUTE_LOCALNET(in_dev)))
                localnet_scope = RT_SCOPE_LINK;

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                if (READ_ONCE(ifa->ifa_flags) & IFA_F_SECONDARY)
                        continue;
                if (min(ifa->ifa_scope, localnet_scope) > scope)
                        continue;
                if (!dst || inet_ifa_match(dst, ifa)) {
                        addr = ifa->ifa_local;
                        break;
                }
                if (!addr)
                        addr = ifa->ifa_local;
        }

        if (addr)
                goto out_unlock;
no_in_dev:
        master_idx = l3mdev_master_ifindex_rcu(dev);

        /* For VRFs, the VRF device takes the place of the loopback device,
         * with addresses on it being preferred.  Note in such cases the
         * loopback device will be among the devices that fail the master_idx
         * equality check in the loop below.
         */
        if (master_idx &&
            (dev = dev_get_by_index_rcu(net, master_idx)) &&
            (in_dev = __in_dev_get_rcu(dev))) {
                addr = in_dev_select_addr(in_dev, scope);
                if (addr)
                        goto out_unlock;
        }

        /* Not loopback addresses on loopback should be preferred
           in this case. It is important that lo is the first interface
           in dev_base list.
         */
        for_each_netdev_rcu(net, dev) {
                if (l3mdev_master_ifindex_rcu(dev) != master_idx)
                        continue;

                in_dev = __in_dev_get_rcu(dev);
                if (!in_dev)
                        continue;

                addr = in_dev_select_addr(in_dev, scope);
                if (addr)
                        goto out_unlock;
        }
out_unlock:
        rcu_read_unlock();
        return addr;
}
EXPORT_SYMBOL(inet_select_addr);

static __be32 confirm_addr_indev(struct in_device *in_dev, __be32 dst,
                              __be32 local, int scope)
{
        unsigned char localnet_scope = RT_SCOPE_HOST;
        const struct in_ifaddr *ifa;
        __be32 addr = 0;
        int same = 0;

        if (unlikely(IN_DEV_ROUTE_LOCALNET(in_dev)))
                localnet_scope = RT_SCOPE_LINK;

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                unsigned char min_scope = min(ifa->ifa_scope, localnet_scope);

                if (!addr &&
                    (local == ifa->ifa_local || !local) &&
                    min_scope <= scope) {
                        addr = ifa->ifa_local;
                        if (same)
                                break;
                }
                if (!same) {
                        same = (!local || inet_ifa_match(local, ifa)) &&
                                (!dst || inet_ifa_match(dst, ifa));
                        if (same && addr) {
                                if (local || !dst)
                                        break;
                                /* Is the selected addr into dst subnet? */
                                if (inet_ifa_match(addr, ifa))
                                        break;
                                /* No, then can we use new local src? */
                                if (min_scope <= scope) {
                                        addr = ifa->ifa_local;
                                        break;
                                }
                                /* search for large dst subnet for addr */
                                same = 0;
                        }
                }
        }

        return same ? addr : 0;
}

/*
 * Confirm that local IP address exists using wildcards:
 * - net: netns to check, cannot be NULL
 * - in_dev: only on this interface, NULL=any interface
 * - dst: only in the same subnet as dst, 0=any dst
 * - local: address, 0=autoselect the local address
 * - scope: maximum allowed scope value for the local address
 */
__be32 inet_confirm_addr(struct net *net, struct in_device *in_dev,
                         __be32 dst, __be32 local, int scope)
{
        __be32 addr = 0;
        struct net_device *dev;

        if (in_dev)
                return confirm_addr_indev(in_dev, dst, local, scope);

        rcu_read_lock();
        for_each_netdev_rcu(net, dev) {
                in_dev = __in_dev_get_rcu(dev);
                if (in_dev) {
                        addr = confirm_addr_indev(in_dev, dst, local, scope);
                        if (addr)
                                break;
                }
        }
        rcu_read_unlock();

        return addr;
}
EXPORT_SYMBOL(inet_confirm_addr);

/*
 *        Device notifier
 */

int register_inetaddr_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&inetaddr_chain, nb);
}
EXPORT_SYMBOL(register_inetaddr_notifier);

int unregister_inetaddr_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&inetaddr_chain, nb);
}
EXPORT_SYMBOL(unregister_inetaddr_notifier);

int register_inetaddr_validator_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&inetaddr_validator_chain, nb);
}
EXPORT_SYMBOL(register_inetaddr_validator_notifier);

int unregister_inetaddr_validator_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&inetaddr_validator_chain,
            nb);
}
EXPORT_SYMBOL(unregister_inetaddr_validator_notifier);

/* Rename ifa_labels for a device name change. Make some effort to preserve
 * existing alias numbering and to create unique labels if possible.
*/
static void inetdev_changename(struct net_device *dev, struct in_device *in_dev)
{
        struct in_ifaddr *ifa;
        int named = 0;

        in_dev_for_each_ifa_rtnl(ifa, in_dev) {
                char old[IFNAMSIZ], *dot;

                memcpy(old, ifa->ifa_label, IFNAMSIZ);
                memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);
                if (named++ == 0)
                        goto skip;
                dot = strchr(old, ':');
                if (!dot) {
                        sprintf(old, ":%d", named);
                        dot = old;
                }
                if (strlen(dot) + strlen(dev->name) < IFNAMSIZ)
                        strcat(ifa->ifa_label, dot);
                else
                        strcpy(ifa->ifa_label + (IFNAMSIZ - strlen(dot) - 1), dot);
skip:
                rtmsg_ifa(RTM_NEWADDR, ifa, NULL, 0);
        }
}

static void inetdev_send_gratuitous_arp(struct net_device *dev,
                                        struct in_device *in_dev)

{
        const struct in_ifaddr *ifa;

        in_dev_for_each_ifa_rtnl(ifa, in_dev) {
                arp_send(ARPOP_REQUEST, ETH_P_ARP,
                         ifa->ifa_local, dev,
                         ifa->ifa_local, NULL,
                         dev->dev_addr, NULL);
        }
}

/* Called only under RTNL semaphore */

static int inetdev_event(struct notifier_block *this, unsigned long event,
                         void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct in_device *in_dev = __in_dev_get_rtnl(dev);

        ASSERT_RTNL();

        if (!in_dev) {
                if (event == NETDEV_REGISTER) {
                        in_dev = inetdev_init(dev);
                        if (IS_ERR(in_dev))
                                return notifier_from_errno(PTR_ERR(in_dev));
                        if (dev->flags & IFF_LOOPBACK) {
                                IN_DEV_CONF_SET(in_dev, NOXFRM, 1);
                                IN_DEV_CONF_SET(in_dev, NOPOLICY, 1);
                        }
                } else if (event == NETDEV_CHANGEMTU) {
                        /* Re-enabling IP */
                        if (inetdev_valid_mtu(dev->mtu))
                                in_dev = inetdev_init(dev);
                }
                goto out;
        }

        switch (event) {
        case NETDEV_REGISTER:
                pr_debug("%s: bug\n", __func__);
                RCU_INIT_POINTER(dev->ip_ptr, NULL);
                break;
        case NETDEV_UP:
                if (!inetdev_valid_mtu(dev->mtu))
                        break;
                if (dev->flags & IFF_LOOPBACK) {
                        struct in_ifaddr *ifa = inet_alloc_ifa(in_dev);

                        if (ifa) {
                                ifa->ifa_local =
                                  ifa->ifa_address = htonl(INADDR_LOOPBACK);
                                ifa->ifa_prefixlen = 8;
                                ifa->ifa_mask = inet_make_mask(8);
                                ifa->ifa_scope = RT_SCOPE_HOST;
                                memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);
                                set_ifa_lifetime(ifa, INFINITY_LIFE_TIME,
                                                 INFINITY_LIFE_TIME);
                                ipv4_devconf_setall(in_dev);
                                neigh_parms_data_state_setall(in_dev->arp_parms);
                                inet_insert_ifa(ifa);
                        }
                }
                ip_mc_up(in_dev);
                fallthrough;
        case NETDEV_CHANGEADDR:
                if (!IN_DEV_ARP_NOTIFY(in_dev))
                        break;
                fallthrough;
        case NETDEV_NOTIFY_PEERS:
                /* Send gratuitous ARP to notify of link change */
                inetdev_send_gratuitous_arp(dev, in_dev);
                break;
        case NETDEV_DOWN:
                ip_mc_down(in_dev);
                break;
        case NETDEV_PRE_TYPE_CHANGE:
                ip_mc_unmap(in_dev);
                break;
        case NETDEV_POST_TYPE_CHANGE:
                ip_mc_remap(in_dev);
                break;
        case NETDEV_CHANGEMTU:
                if (inetdev_valid_mtu(dev->mtu))
                        break;
                /* disable IP when MTU is not enough */
                fallthrough;
        case NETDEV_UNREGISTER:
                inetdev_destroy(in_dev);
                break;
        case NETDEV_CHANGENAME:
                /* Do not notify about label change, this event is
                 * not interesting to applications using netlink.
                 */
                inetdev_changename(dev, in_dev);

                devinet_sysctl_unregister(in_dev);
                devinet_sysctl_register(in_dev);
                break;
        }
out:
        return NOTIFY_DONE;
}

static struct notifier_block ip_netdev_notifier = {
        .notifier_call = inetdev_event,
};

static size_t inet_nlmsg_size(void)
{
        return NLMSG_ALIGN(sizeof(struct ifaddrmsg))
               + nla_total_size(4) /* IFA_ADDRESS */
               + nla_total_size(4) /* IFA_LOCAL */
               + nla_total_size(4) /* IFA_BROADCAST */
               + nla_total_size(IFNAMSIZ) /* IFA_LABEL */
               + nla_total_size(4)  /* IFA_FLAGS */
               + nla_total_size(1)  /* IFA_PROTO */
               + nla_total_size(4)  /* IFA_RT_PRIORITY */
               + nla_total_size(sizeof(struct ifa_cacheinfo)); /* IFA_CACHEINFO */
}

static inline u32 cstamp_delta(unsigned long cstamp)
{
        return (cstamp - INITIAL_JIFFIES) * 100UL / HZ;
}

static int put_cacheinfo(struct sk_buff *skb, unsigned long cstamp,
                         unsigned long tstamp, u32 preferred, u32 valid)
{
        struct ifa_cacheinfo ci;

        ci.cstamp = cstamp_delta(cstamp);
        ci.tstamp = cstamp_delta(tstamp);
        ci.ifa_prefered = preferred;
        ci.ifa_valid = valid;

        return nla_put(skb, IFA_CACHEINFO, sizeof(ci), &ci);
}

static int inet_fill_ifaddr(struct sk_buff *skb, const struct in_ifaddr *ifa,
                            struct inet_fill_args *args)
{
        struct ifaddrmsg *ifm;
        struct nlmsghdr  *nlh;
        unsigned long tstamp;
        u32 preferred, valid;
        u32 flags;

        nlh = nlmsg_put(skb, args->portid, args->seq, args->event, sizeof(*ifm),
                        args->flags);
        if (!nlh)
                return -EMSGSIZE;

        ifm = nlmsg_data(nlh);
        ifm->ifa_family = AF_INET;
        ifm->ifa_prefixlen = ifa->ifa_prefixlen;

        flags = READ_ONCE(ifa->ifa_flags);
        /* Warning : ifm->ifa_flags is an __u8, it holds only 8 bits.
         * The 32bit value is given in IFA_FLAGS attribute.
         */
        ifm->ifa_flags = (__u8)flags;

        ifm->ifa_scope = ifa->ifa_scope;
        ifm->ifa_index = ifa->ifa_dev->dev->ifindex;

        if (args->netnsid >= 0 &&
            nla_put_s32(skb, IFA_TARGET_NETNSID, args->netnsid))
                goto nla_put_failure;

        tstamp = READ_ONCE(ifa->ifa_tstamp);
        if (!(flags & IFA_F_PERMANENT)) {
                preferred = READ_ONCE(ifa->ifa_preferred_lft);
                valid = READ_ONCE(ifa->ifa_valid_lft);
                if (preferred != INFINITY_LIFE_TIME) {
                        long tval = (jiffies - tstamp) / HZ;

                        if (preferred > tval)
                                preferred -= tval;
                        else
                                preferred = 0;
                        if (valid != INFINITY_LIFE_TIME) {
                                if (valid > tval)
                                        valid -= tval;
                                else
                                        valid = 0;
                        }
                }
        } else {
                preferred = INFINITY_LIFE_TIME;
                valid = INFINITY_LIFE_TIME;
        }
        if ((ifa->ifa_address &&
             nla_put_in_addr(skb, IFA_ADDRESS, ifa->ifa_address)) ||
            (ifa->ifa_local &&
             nla_put_in_addr(skb, IFA_LOCAL, ifa->ifa_local)) ||
            (ifa->ifa_broadcast &&
             nla_put_in_addr(skb, IFA_BROADCAST, ifa->ifa_broadcast)) ||
            (ifa->ifa_label[0] &&
             nla_put_string(skb, IFA_LABEL, ifa->ifa_label)) ||
            (ifa->ifa_proto &&
             nla_put_u8(skb, IFA_PROTO, ifa->ifa_proto)) ||
            nla_put_u32(skb, IFA_FLAGS, flags) ||
            (ifa->ifa_rt_priority &&
             nla_put_u32(skb, IFA_RT_PRIORITY, ifa->ifa_rt_priority)) ||
            put_cacheinfo(skb, READ_ONCE(ifa->ifa_cstamp), tstamp,
                          preferred, valid))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int inet_valid_dump_ifaddr_req(const struct nlmsghdr *nlh,
                                      struct inet_fill_args *fillargs,
                                      struct net **tgt_net, struct sock *sk,
                                      struct netlink_callback *cb)
{
        struct netlink_ext_ack *extack = cb->extack;
        struct nlattr *tb[IFA_MAX+1];
        struct ifaddrmsg *ifm;
        int err, i;

        ifm = nlmsg_payload(nlh, sizeof(*ifm));
        if (!ifm) {
                NL_SET_ERR_MSG(extack, "ipv4: Invalid header for address dump request");
                return -EINVAL;
        }

        if (ifm->ifa_prefixlen || ifm->ifa_flags || ifm->ifa_scope) {
                NL_SET_ERR_MSG(extack, "ipv4: Invalid values in header for address dump request");
                return -EINVAL;
        }

        fillargs->ifindex = ifm->ifa_index;
        if (fillargs->ifindex) {
                cb->answer_flags |= NLM_F_DUMP_FILTERED;
                fillargs->flags |= NLM_F_DUMP_FILTERED;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(*ifm), tb, IFA_MAX,
                                            ifa_ipv4_policy, extack);
        if (err < 0)
                return err;

        for (i = 0; i <= IFA_MAX; ++i) {
                if (!tb[i])
                        continue;

                if (i == IFA_TARGET_NETNSID) {
                        struct net *net;

                        fillargs->netnsid = nla_get_s32(tb[i]);

                        net = rtnl_get_net_ns_capable(sk, fillargs->netnsid);
                        if (IS_ERR(net)) {
                                fillargs->netnsid = -1;
                                NL_SET_ERR_MSG(extack, "ipv4: Invalid target network namespace id");
                                return PTR_ERR(net);
                        }
                        *tgt_net = net;
                } else {
                        NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in dump request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int in_dev_dump_ifmcaddr(struct in_device *in_dev, struct sk_buff *skb,
                                struct netlink_callback *cb, int *s_ip_idx,
                                struct inet_fill_args *fillargs)
{
        struct ip_mc_list *im;
        int ip_idx = 0;
        int err;

        for (im = rcu_dereference(in_dev->mc_list);
             im;
             im = rcu_dereference(im->next_rcu)) {
                if (ip_idx < *s_ip_idx) {
                        ip_idx++;
                        continue;
                }
                err = inet_fill_ifmcaddr(skb, in_dev->dev, im, fillargs);
                if (err < 0)
                        goto done;

                nl_dump_check_consistent(cb, nlmsg_hdr(skb));
                ip_idx++;
        }
        err = 0;
        ip_idx = 0;
done:
        *s_ip_idx = ip_idx;
        return err;
}

static int in_dev_dump_ifaddr(struct in_device *in_dev, struct sk_buff *skb,
                              struct netlink_callback *cb, int *s_ip_idx,
                              struct inet_fill_args *fillargs)
{
        struct in_ifaddr *ifa;
        int ip_idx = 0;
        int err;

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                if (ip_idx < *s_ip_idx) {
                        ip_idx++;
                        continue;
                }
                err = inet_fill_ifaddr(skb, ifa, fillargs);
                if (err < 0)
                        goto done;

                nl_dump_check_consistent(cb, nlmsg_hdr(skb));
                ip_idx++;
        }
        err = 0;
        ip_idx = 0;
done:
        *s_ip_idx = ip_idx;

        return err;
}

static int in_dev_dump_addr(struct in_device *in_dev, struct sk_buff *skb,
                            struct netlink_callback *cb, int *s_ip_idx,
                            struct inet_fill_args *fillargs)
{
        switch (fillargs->event) {
        case RTM_NEWADDR:
                return in_dev_dump_ifaddr(in_dev, skb, cb, s_ip_idx, fillargs);
        case RTM_GETMULTICAST:
                return in_dev_dump_ifmcaddr(in_dev, skb, cb, s_ip_idx,
                                            fillargs);
        default:
                return -EINVAL;
        }
}

/* Combine dev_addr_genid and dev_base_seq to detect changes.
 */
static u32 inet_base_seq(const struct net *net)
{
        u32 res = atomic_read(&net->ipv4.dev_addr_genid) +
                  READ_ONCE(net->dev_base_seq);

        /* Must not return 0 (see nl_dump_check_consistent()).
         * Chose a value far away from 0.
         */
        if (!res)
                res = 0x80000000;
        return res;
}

static int inet_dump_addr(struct sk_buff *skb, struct netlink_callback *cb,
                          int event)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct inet_fill_args fillargs = {
                .portid = NETLINK_CB(cb->skb).portid,
                .seq = nlh->nlmsg_seq,
                .event = event,
                .flags = NLM_F_MULTI,
                .netnsid = -1,
        };
        struct net *net = sock_net(skb->sk);
        struct net *tgt_net = net;
        struct {
                unsigned long ifindex;
                int ip_idx;
        } *ctx = (void *)cb->ctx;
        struct in_device *in_dev;
        struct net_device *dev;
        int err = 0;

        rcu_read_lock();
        if (cb->strict_check) {
                err = inet_valid_dump_ifaddr_req(nlh, &fillargs, &tgt_net,
                                                 skb->sk, cb);
                if (err < 0)
                        goto done;

                if (fillargs.ifindex) {
                        dev = dev_get_by_index_rcu(tgt_net, fillargs.ifindex);
                        if (!dev) {
                                err = -ENODEV;
                                goto done;
                        }
                        in_dev = __in_dev_get_rcu(dev);
                        if (!in_dev)
                                goto done;
                        err = in_dev_dump_addr(in_dev, skb, cb, &ctx->ip_idx,
                                               &fillargs);
                        goto done;
                }
        }

        cb->seq = inet_base_seq(tgt_net);

        for_each_netdev_dump(tgt_net, dev, ctx->ifindex) {
                in_dev = __in_dev_get_rcu(dev);
                if (!in_dev)
                        continue;
                err = in_dev_dump_addr(in_dev, skb, cb, &ctx->ip_idx,
                                       &fillargs);
                if (err < 0)
                        goto done;
        }
done:
        if (fillargs.netnsid >= 0)
                put_net(tgt_net);
        rcu_read_unlock();
        return err;
}

static int inet_dump_ifaddr(struct sk_buff *skb, struct netlink_callback *cb)
{
        return inet_dump_addr(skb, cb, RTM_NEWADDR);
}

static int inet_dump_ifmcaddr(struct sk_buff *skb, struct netlink_callback *cb)
{
        return inet_dump_addr(skb, cb, RTM_GETMULTICAST);
}

static void rtmsg_ifa(int event, struct in_ifaddr *ifa, struct nlmsghdr *nlh,
                      u32 portid)
{
        struct inet_fill_args fillargs = {
                .portid = portid,
                .seq = nlh ? nlh->nlmsg_seq : 0,
                .event = event,
                .flags = 0,
                .netnsid = -1,
        };
        struct sk_buff *skb;
        int err = -ENOBUFS;
        struct net *net;

        net = dev_net(ifa->ifa_dev->dev);
        skb = nlmsg_new(inet_nlmsg_size(), GFP_KERNEL);
        if (!skb)
                goto errout;

        err = inet_fill_ifaddr(skb, ifa, &fillargs);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in inet_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, portid, RTNLGRP_IPV4_IFADDR, nlh, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_IPV4_IFADDR, err);
}

static size_t inet_get_link_af_size(const struct net_device *dev,
                                    u32 ext_filter_mask)
{
        struct in_device *in_dev = rcu_dereference_rtnl(dev->ip_ptr);

        if (!in_dev)
                return 0;

        return nla_total_size(IPV4_DEVCONF_MAX * 4); /* IFLA_INET_CONF */
}

static int inet_fill_link_af(struct sk_buff *skb, const struct net_device *dev,
                             u32 ext_filter_mask)
{
        struct in_device *in_dev = rcu_dereference_rtnl(dev->ip_ptr);
        struct nlattr *nla;
        int i;

        if (!in_dev)
                return -ENODATA;

        nla = nla_reserve(skb, IFLA_INET_CONF, IPV4_DEVCONF_MAX * 4);
        if (!nla)
                return -EMSGSIZE;

        for (i = 0; i < IPV4_DEVCONF_MAX; i++)
                ((u32 *) nla_data(nla))[i] = READ_ONCE(in_dev->cnf.data[i]);

        return 0;
}

static const struct nla_policy inet_af_policy[IFLA_INET_MAX+1] = {
        [IFLA_INET_CONF]        = { .type = NLA_NESTED },
};

static const struct nla_policy inet_devconf_policy[IPV4_DEVCONF_MAX + 1] = {
        [IPV4_DEVCONF_FORWARDING]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_MC_FORWARDING]        = { .type = NLA_REJECT },
        [IPV4_DEVCONF_PROXY_ARP]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_ACCEPT_REDIRECTS]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_SECURE_REDIRECTS]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_SEND_REDIRECTS]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_SHARED_MEDIA]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_RP_FILTER]        = NLA_POLICY_RANGE(NLA_U32, 0, 2),
        [IPV4_DEVCONF_ACCEPT_SOURCE_ROUTE] = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_BOOTP_RELAY]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_LOG_MARTIANS]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_TAG]                = { .type = NLA_U32 },
        [IPV4_DEVCONF_ARPFILTER]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_MEDIUM_ID]        = NLA_POLICY_MIN(NLA_S32, -1),
        [IPV4_DEVCONF_NOXFRM]                = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_NOPOLICY]                = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_FORCE_IGMP_VERSION] = NLA_POLICY_RANGE(NLA_U32, 0, 3),
        [IPV4_DEVCONF_ARP_ANNOUNCE]        = NLA_POLICY_RANGE(NLA_U32, 0, 2),
        [IPV4_DEVCONF_ARP_IGNORE]        = NLA_POLICY_RANGE(NLA_U32, 0, 8),
        [IPV4_DEVCONF_PROMOTE_SECONDARIES] = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_ARP_ACCEPT]        = NLA_POLICY_RANGE(NLA_U32, 0, 2),
        [IPV4_DEVCONF_ARP_NOTIFY]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_ACCEPT_LOCAL]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_SRC_VMARK]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_PROXY_ARP_PVLAN]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_ROUTE_LOCALNET]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_BC_FORWARDING]        = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_IGMPV2_UNSOLICITED_REPORT_INTERVAL] = { .type = NLA_U32 },
        [IPV4_DEVCONF_IGMPV3_UNSOLICITED_REPORT_INTERVAL] = { .type = NLA_U32 },
        [IPV4_DEVCONF_IGNORE_ROUTES_WITH_LINKDOWN] =
                NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_DROP_UNICAST_IN_L2_MULTICAST] =
                NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_DROP_GRATUITOUS_ARP] = NLA_POLICY_RANGE(NLA_U32, 0, 1),
        [IPV4_DEVCONF_ARP_EVICT_NOCARRIER] = NLA_POLICY_RANGE(NLA_U32, 0, 1),
};

static int inet_validate_link_af(const struct net_device *dev,
                                 const struct nlattr *nla,
                                 struct netlink_ext_ack *extack)
{
        struct nlattr *tb[IFLA_INET_MAX + 1], *nested_tb[IPV4_DEVCONF_MAX + 1];
        int err;

        if (dev && !__in_dev_get_rtnl(dev))
                return -EAFNOSUPPORT;

        err = nla_parse_nested_deprecated(tb, IFLA_INET_MAX, nla,
                                          inet_af_policy, extack);
        if (err < 0)
                return err;

        if (tb[IFLA_INET_CONF]) {
                err = nla_parse_nested(nested_tb, IPV4_DEVCONF_MAX,
                                       tb[IFLA_INET_CONF], inet_devconf_policy,
                                       extack);

                if (err < 0)
                        return err;
        }

        return 0;
}

static int inet_set_link_af(struct net_device *dev, const struct nlattr *nla,
                            struct netlink_ext_ack *extack)
{
        struct in_device *in_dev = __in_dev_get_rtnl(dev);
        struct nlattr *a, *tb[IFLA_INET_MAX+1];
        int rem;

        if (!in_dev)
                return -EAFNOSUPPORT;

        if (nla_parse_nested_deprecated(tb, IFLA_INET_MAX, nla, NULL, NULL) < 0)
                return -EINVAL;

        if (tb[IFLA_INET_CONF]) {
                nla_for_each_nested(a, tb[IFLA_INET_CONF], rem)
                        ipv4_devconf_set(in_dev, nla_type(a), nla_get_u32(a));
        }

        return 0;
}

static int inet_netconf_msgsize_devconf(int type)
{
        int size = NLMSG_ALIGN(sizeof(struct netconfmsg))
                   + nla_total_size(4);        /* NETCONFA_IFINDEX */
        bool all = false;

        if (type == NETCONFA_ALL)
                all = true;

        if (all || type == NETCONFA_FORWARDING)
                size += nla_total_size(4);
        if (all || type == NETCONFA_RP_FILTER)
                size += nla_total_size(4);
        if (all || type == NETCONFA_MC_FORWARDING)
                size += nla_total_size(4);
        if (all || type == NETCONFA_BC_FORWARDING)
                size += nla_total_size(4);
        if (all || type == NETCONFA_PROXY_NEIGH)
                size += nla_total_size(4);
        if (all || type == NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN)
                size += nla_total_size(4);

        return size;
}

static int inet_netconf_fill_devconf(struct sk_buff *skb, int ifindex,
                                     const struct ipv4_devconf *devconf,
                                     u32 portid, u32 seq, int event,
                                     unsigned int flags, int type)
{
        struct nlmsghdr  *nlh;
        struct netconfmsg *ncm;
        bool all = false;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct netconfmsg),
                        flags);
        if (!nlh)
                return -EMSGSIZE;

        if (type == NETCONFA_ALL)
                all = true;

        ncm = nlmsg_data(nlh);
        ncm->ncm_family = AF_INET;

        if (nla_put_s32(skb, NETCONFA_IFINDEX, ifindex) < 0)
                goto nla_put_failure;

        if (!devconf)
                goto out;

        if ((all || type == NETCONFA_FORWARDING) &&
            nla_put_s32(skb, NETCONFA_FORWARDING,
                        IPV4_DEVCONF_RO(*devconf, FORWARDING)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_RP_FILTER) &&
            nla_put_s32(skb, NETCONFA_RP_FILTER,
                        IPV4_DEVCONF_RO(*devconf, RP_FILTER)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_MC_FORWARDING) &&
            nla_put_s32(skb, NETCONFA_MC_FORWARDING,
                        IPV4_DEVCONF_RO(*devconf, MC_FORWARDING)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_BC_FORWARDING) &&
            nla_put_s32(skb, NETCONFA_BC_FORWARDING,
                        IPV4_DEVCONF_RO(*devconf, BC_FORWARDING)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_PROXY_NEIGH) &&
            nla_put_s32(skb, NETCONFA_PROXY_NEIGH,
                        IPV4_DEVCONF_RO(*devconf, PROXY_ARP)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN) &&
            nla_put_s32(skb, NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN,
                        IPV4_DEVCONF_RO(*devconf,
                                        IGNORE_ROUTES_WITH_LINKDOWN)) < 0)
                goto nla_put_failure;

out:
        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

void inet_netconf_notify_devconf(struct net *net, int event, int type,
                                 int ifindex, struct ipv4_devconf *devconf)
{
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(inet_netconf_msgsize_devconf(type), GFP_KERNEL);
        if (!skb)
                goto errout;

        err = inet_netconf_fill_devconf(skb, ifindex, devconf, 0, 0,
                                        event, 0, type);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in inet_netconf_msgsize_devconf() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, 0, RTNLGRP_IPV4_NETCONF, NULL, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_IPV4_NETCONF, err);
}

static const struct nla_policy devconf_ipv4_policy[NETCONFA_MAX+1] = {
        [NETCONFA_IFINDEX]        = { .len = sizeof(int) },
        [NETCONFA_FORWARDING]        = { .len = sizeof(int) },
        [NETCONFA_RP_FILTER]        = { .len = sizeof(int) },
        [NETCONFA_PROXY_NEIGH]        = { .len = sizeof(int) },
        [NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN]        = { .len = sizeof(int) },
};

static int inet_netconf_valid_get_req(struct sk_buff *skb,
                                      const struct nlmsghdr *nlh,
                                      struct nlattr **tb,
                                      struct netlink_ext_ack *extack)
{
        int i, err;

        if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(struct netconfmsg))) {
                NL_SET_ERR_MSG(extack, "ipv4: Invalid header for netconf get request");
                return -EINVAL;
        }

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(struct netconfmsg),
                                              tb, NETCONFA_MAX,
                                              devconf_ipv4_policy, extack);

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct netconfmsg),
                                            tb, NETCONFA_MAX,
                                            devconf_ipv4_policy, extack);
        if (err)
                return err;

        for (i = 0; i <= NETCONFA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case NETCONFA_IFINDEX:
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in netconf get request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int inet_netconf_get_devconf(struct sk_buff *in_skb,
                                    struct nlmsghdr *nlh,
                                    struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        struct nlattr *tb[NETCONFA_MAX + 1];
        const struct ipv4_devconf *devconf;
        struct in_device *in_dev = NULL;
        struct net_device *dev = NULL;
        struct sk_buff *skb;
        int ifindex;
        int err;

        err = inet_netconf_valid_get_req(in_skb, nlh, tb, extack);
        if (err)
                return err;

        if (!tb[NETCONFA_IFINDEX])
                return -EINVAL;

        ifindex = nla_get_s32(tb[NETCONFA_IFINDEX]);
        switch (ifindex) {
        case NETCONFA_IFINDEX_ALL:
                devconf = net->ipv4.devconf_all;
                break;
        case NETCONFA_IFINDEX_DEFAULT:
                devconf = net->ipv4.devconf_dflt;
                break;
        default:
                err = -ENODEV;
                dev = dev_get_by_index(net, ifindex);
                if (dev)
                        in_dev = in_dev_get(dev);
                if (!in_dev)
                        goto errout;
                devconf = &in_dev->cnf;
                break;
        }

        err = -ENOBUFS;
        skb = nlmsg_new(inet_netconf_msgsize_devconf(NETCONFA_ALL), GFP_KERNEL);
        if (!skb)
                goto errout;

        err = inet_netconf_fill_devconf(skb, ifindex, devconf,
                                        NETLINK_CB(in_skb).portid,
                                        nlh->nlmsg_seq, RTM_NEWNETCONF, 0,
                                        NETCONFA_ALL);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in inet_netconf_msgsize_devconf() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
errout:
        if (in_dev)
                in_dev_put(in_dev);
        dev_put(dev);
        return err;
}

static int inet_netconf_dump_devconf(struct sk_buff *skb,
                                     struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        struct {
                unsigned long ifindex;
                unsigned int all_default;
        } *ctx = (void *)cb->ctx;
        const struct in_device *in_dev;
        struct net_device *dev;
        int err = 0;

        if (cb->strict_check) {
                struct netlink_ext_ack *extack = cb->extack;
                struct netconfmsg *ncm;

                if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ncm))) {
                        NL_SET_ERR_MSG(extack, "ipv4: Invalid header for netconf dump request");
                        return -EINVAL;
                }

                if (nlmsg_attrlen(nlh, sizeof(*ncm))) {
                        NL_SET_ERR_MSG(extack, "ipv4: Invalid data after header in netconf dump request");
                        return -EINVAL;
                }
        }

        rcu_read_lock();
        for_each_netdev_dump(net, dev, ctx->ifindex) {
                in_dev = __in_dev_get_rcu(dev);
                if (!in_dev)
                        continue;
                err = inet_netconf_fill_devconf(skb, dev->ifindex,
                                                &in_dev->cnf,
                                                NETLINK_CB(cb->skb).portid,
                                                nlh->nlmsg_seq,
                                                RTM_NEWNETCONF, NLM_F_MULTI,
                                                NETCONFA_ALL);
                if (err < 0)
                        goto done;
        }
        if (ctx->all_default == 0) {
                err = inet_netconf_fill_devconf(skb, NETCONFA_IFINDEX_ALL,
                                                net->ipv4.devconf_all,
                                                NETLINK_CB(cb->skb).portid,
                                                nlh->nlmsg_seq,
                                                RTM_NEWNETCONF, NLM_F_MULTI,
                                                NETCONFA_ALL);
                if (err < 0)
                        goto done;
                ctx->all_default++;
        }
        if (ctx->all_default == 1) {
                err = inet_netconf_fill_devconf(skb, NETCONFA_IFINDEX_DEFAULT,
                                                net->ipv4.devconf_dflt,
                                                NETLINK_CB(cb->skb).portid,
                                                nlh->nlmsg_seq,
                                                RTM_NEWNETCONF, NLM_F_MULTI,
                                                NETCONFA_ALL);
                if (err < 0)
                        goto done;
                ctx->all_default++;
        }
done:
        rcu_read_unlock();
        return err;
}

#ifdef CONFIG_SYSCTL

static void devinet_copy_dflt_conf(struct net *net, int i)
{
        struct net_device *dev;

        rcu_read_lock();
        for_each_netdev_rcu(net, dev) {
                struct in_device *in_dev;

                in_dev = __in_dev_get_rcu(dev);
                if (in_dev && !test_bit(i, in_dev->cnf.state))
                        in_dev->cnf.data[i] = net->ipv4.devconf_dflt->data[i];
        }
        rcu_read_unlock();
}

/* called with RTNL locked */
static void inet_forward_change(struct net *net)
{
        struct net_device *dev;
        int on = IPV4_DEVCONF_ALL(net, FORWARDING);

        IPV4_DEVCONF_ALL(net, ACCEPT_REDIRECTS) = !on;
        IPV4_DEVCONF_DFLT(net, FORWARDING) = on;
        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                    NETCONFA_FORWARDING,
                                    NETCONFA_IFINDEX_ALL,
                                    net->ipv4.devconf_all);
        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                    NETCONFA_FORWARDING,
                                    NETCONFA_IFINDEX_DEFAULT,
                                    net->ipv4.devconf_dflt);

        for_each_netdev(net, dev) {
                struct in_device *in_dev;

                if (on)
                        dev_disable_lro(dev);

                in_dev = __in_dev_get_rtnl_net(dev);
                if (in_dev) {
                        IN_DEV_CONF_SET(in_dev, FORWARDING, on);
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_FORWARDING,
                                                    dev->ifindex, &in_dev->cnf);
                }
        }
}

static int devinet_conf_ifindex(struct net *net, struct ipv4_devconf *cnf)
{
        if (cnf == net->ipv4.devconf_dflt)
                return NETCONFA_IFINDEX_DEFAULT;
        else if (cnf == net->ipv4.devconf_all)
                return NETCONFA_IFINDEX_ALL;
        else {
                struct in_device *idev
                        = container_of(cnf, struct in_device, cnf);
                return idev->dev->ifindex;
        }
}

static int devinet_conf_proc(const struct ctl_table *ctl, int write,
                             void *buffer, size_t *lenp, loff_t *ppos)
{
        int old_value = *(int *)ctl->data;
        int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
        int new_value = *(int *)ctl->data;

        if (write) {
                struct ipv4_devconf *cnf = ctl->extra1;
                struct net *net = ctl->extra2;
                int i = (int *)ctl->data - cnf->data;
                int ifindex;

                set_bit(i, cnf->state);

                if (cnf == net->ipv4.devconf_dflt)
                        devinet_copy_dflt_conf(net, i);
                if (i == IPV4_DEVCONF_ACCEPT_LOCAL - 1 ||
                    i == IPV4_DEVCONF_ROUTE_LOCALNET - 1)
                        if ((new_value == 0) && (old_value != 0))
                                rt_cache_flush(net);

                if (i == IPV4_DEVCONF_BC_FORWARDING - 1 &&
                    new_value != old_value)
                        rt_cache_flush(net);

                if (i == IPV4_DEVCONF_RP_FILTER - 1 &&
                    new_value != old_value) {
                        ifindex = devinet_conf_ifindex(net, cnf);
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_RP_FILTER,
                                                    ifindex, cnf);
                }
                if (i == IPV4_DEVCONF_PROXY_ARP - 1 &&
                    new_value != old_value) {
                        ifindex = devinet_conf_ifindex(net, cnf);
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_PROXY_NEIGH,
                                                    ifindex, cnf);
                }
                if (i == IPV4_DEVCONF_IGNORE_ROUTES_WITH_LINKDOWN - 1 &&
                    new_value != old_value) {
                        ifindex = devinet_conf_ifindex(net, cnf);
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN,
                                                    ifindex, cnf);
                }
        }

        return ret;
}

static int devinet_sysctl_forward(const struct ctl_table *ctl, int write,
                                  void *buffer, size_t *lenp, loff_t *ppos)
{
        int *valp = ctl->data;
        int val = *valp;
        loff_t pos = *ppos;
        struct net *net = ctl->extra2;
        int ret;

        if (write && !ns_capable(net->user_ns, CAP_NET_ADMIN))
                return -EPERM;

        ret = proc_dointvec(ctl, write, buffer, lenp, ppos);

        if (write && *valp != val) {
                if (valp != &IPV4_DEVCONF_DFLT(net, FORWARDING)) {
                        if (!rtnl_net_trylock(net)) {
                                /* Restore the original values before restarting */
                                *valp = val;
                                *ppos = pos;
                                return restart_syscall();
                        }
                        if (valp == &IPV4_DEVCONF_ALL(net, FORWARDING)) {
                                inet_forward_change(net);
                        } else {
                                struct ipv4_devconf *cnf = ctl->extra1;
                                struct in_device *idev =
                                        container_of(cnf, struct in_device, cnf);
                                if (*valp)
                                        dev_disable_lro(idev->dev);
                                inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                            NETCONFA_FORWARDING,
                                                            idev->dev->ifindex,
                                                            cnf);
                        }
                        rtnl_net_unlock(net);
                        rt_cache_flush(net);
                } else
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_FORWARDING,
                                                    NETCONFA_IFINDEX_DEFAULT,
                                                    net->ipv4.devconf_dflt);
        }

        return ret;
}

static int ipv4_doint_and_flush(const struct ctl_table *ctl, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        int *valp = ctl->data;
        int val = *valp;
        int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
        struct net *net = ctl->extra2;

        if (write && *valp != val)
                rt_cache_flush(net);

        return ret;
}

#define DEVINET_SYSCTL_ENTRY(attr, name, mval, proc) \
        { \
                .procname        = name, \
                .data                = ipv4_devconf.data + \
                                  IPV4_DEVCONF_ ## attr - 1, \
                .maxlen                = sizeof(int), \
                .mode                = mval, \
                .proc_handler        = proc, \
                .extra1                = &ipv4_devconf, \
        }

#define DEVINET_SYSCTL_RW_ENTRY(attr, name) \
        DEVINET_SYSCTL_ENTRY(attr, name, 0644, devinet_conf_proc)

#define DEVINET_SYSCTL_RO_ENTRY(attr, name) \
        DEVINET_SYSCTL_ENTRY(attr, name, 0444, devinet_conf_proc)

#define DEVINET_SYSCTL_COMPLEX_ENTRY(attr, name, proc) \
        DEVINET_SYSCTL_ENTRY(attr, name, 0644, proc)

#define DEVINET_SYSCTL_FLUSHING_ENTRY(attr, name) \
        DEVINET_SYSCTL_COMPLEX_ENTRY(attr, name, ipv4_doint_and_flush)

static struct devinet_sysctl_table {
        struct ctl_table_header *sysctl_header;
        struct ctl_table devinet_vars[IPV4_DEVCONF_MAX];
} devinet_sysctl = {
        .devinet_vars = {
                DEVINET_SYSCTL_COMPLEX_ENTRY(FORWARDING, "forwarding",
                                             devinet_sysctl_forward),
                DEVINET_SYSCTL_RO_ENTRY(MC_FORWARDING, "mc_forwarding"),
                DEVINET_SYSCTL_RW_ENTRY(BC_FORWARDING, "bc_forwarding"),

                DEVINET_SYSCTL_RW_ENTRY(ACCEPT_REDIRECTS, "accept_redirects"),
                DEVINET_SYSCTL_RW_ENTRY(SECURE_REDIRECTS, "secure_redirects"),
                DEVINET_SYSCTL_RW_ENTRY(SHARED_MEDIA, "shared_media"),
                DEVINET_SYSCTL_RW_ENTRY(RP_FILTER, "rp_filter"),
                DEVINET_SYSCTL_RW_ENTRY(SEND_REDIRECTS, "send_redirects"),
                DEVINET_SYSCTL_RW_ENTRY(ACCEPT_SOURCE_ROUTE,
                                        "accept_source_route"),
                DEVINET_SYSCTL_RW_ENTRY(ACCEPT_LOCAL, "accept_local"),
                DEVINET_SYSCTL_RW_ENTRY(SRC_VMARK, "src_valid_mark"),
                DEVINET_SYSCTL_RW_ENTRY(PROXY_ARP, "proxy_arp"),
                DEVINET_SYSCTL_RW_ENTRY(MEDIUM_ID, "medium_id"),
                DEVINET_SYSCTL_RW_ENTRY(BOOTP_RELAY, "bootp_relay"),
                DEVINET_SYSCTL_RW_ENTRY(LOG_MARTIANS, "log_martians"),
                DEVINET_SYSCTL_RW_ENTRY(TAG, "tag"),
                DEVINET_SYSCTL_RW_ENTRY(ARPFILTER, "arp_filter"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_ANNOUNCE, "arp_announce"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_IGNORE, "arp_ignore"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_ACCEPT, "arp_accept"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_NOTIFY, "arp_notify"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_EVICT_NOCARRIER,
                                        "arp_evict_nocarrier"),
                DEVINET_SYSCTL_RW_ENTRY(PROXY_ARP_PVLAN, "proxy_arp_pvlan"),
                DEVINET_SYSCTL_RW_ENTRY(FORCE_IGMP_VERSION,
                                        "force_igmp_version"),
                DEVINET_SYSCTL_RW_ENTRY(IGMPV2_UNSOLICITED_REPORT_INTERVAL,
                                        "igmpv2_unsolicited_report_interval"),
                DEVINET_SYSCTL_RW_ENTRY(IGMPV3_UNSOLICITED_REPORT_INTERVAL,
                                        "igmpv3_unsolicited_report_interval"),
                DEVINET_SYSCTL_RW_ENTRY(IGNORE_ROUTES_WITH_LINKDOWN,
                                        "ignore_routes_with_linkdown"),
                DEVINET_SYSCTL_RW_ENTRY(DROP_GRATUITOUS_ARP,
                                        "drop_gratuitous_arp"),

                DEVINET_SYSCTL_FLUSHING_ENTRY(NOXFRM, "disable_xfrm"),
                DEVINET_SYSCTL_FLUSHING_ENTRY(NOPOLICY, "disable_policy"),
                DEVINET_SYSCTL_FLUSHING_ENTRY(PROMOTE_SECONDARIES,
                                              "promote_secondaries"),
                DEVINET_SYSCTL_FLUSHING_ENTRY(ROUTE_LOCALNET,
                                              "route_localnet"),
                DEVINET_SYSCTL_FLUSHING_ENTRY(DROP_UNICAST_IN_L2_MULTICAST,
                                              "drop_unicast_in_l2_multicast"),
        },
};

static int __devinet_sysctl_register(struct net *net, char *dev_name,
                                     int ifindex, struct ipv4_devconf *p)
{
        int i;
        struct devinet_sysctl_table *t;
        char path[sizeof("net/ipv4/conf/") + IFNAMSIZ];

        t = kmemdup(&devinet_sysctl, sizeof(*t), GFP_KERNEL_ACCOUNT);
        if (!t)
                goto out;

        for (i = 0; i < ARRAY_SIZE(t->devinet_vars); i++) {
                t->devinet_vars[i].data += (char *)p - (char *)&ipv4_devconf;
                t->devinet_vars[i].extra1 = p;
                t->devinet_vars[i].extra2 = net;
        }

        snprintf(path, sizeof(path), "net/ipv4/conf/%s", dev_name);

        t->sysctl_header = register_net_sysctl(net, path, t->devinet_vars);
        if (!t->sysctl_header)
                goto free;

        p->sysctl = t;

        inet_netconf_notify_devconf(net, RTM_NEWNETCONF, NETCONFA_ALL,
                                    ifindex, p);
        return 0;

free:
        kfree(t);
out:
        return -ENOMEM;
}

static void __devinet_sysctl_unregister(struct net *net,
                                        struct ipv4_devconf *cnf, int ifindex)
{
        struct devinet_sysctl_table *t = cnf->sysctl;

        if (t) {
                cnf->sysctl = NULL;
                unregister_net_sysctl_table(t->sysctl_header);
                kfree(t);
        }

        inet_netconf_notify_devconf(net, RTM_DELNETCONF, 0, ifindex, NULL);
}

static int devinet_sysctl_register(struct in_device *idev)
{
        int err;

        if (!sysctl_dev_name_is_allowed(idev->dev->name))
                return -EINVAL;

        err = neigh_sysctl_register(idev->dev, idev->arp_parms, NULL);
        if (err)
                return err;
        err = __devinet_sysctl_register(dev_net(idev->dev), idev->dev->name,
                                        idev->dev->ifindex, &idev->cnf);
        if (err)
                neigh_sysctl_unregister(idev->arp_parms);
        return err;
}

static void devinet_sysctl_unregister(struct in_device *idev)
{
        struct net *net = dev_net(idev->dev);

        __devinet_sysctl_unregister(net, &idev->cnf, idev->dev->ifindex);
        neigh_sysctl_unregister(idev->arp_parms);
}

static struct ctl_table ctl_forward_entry[] = {
        {
                .procname        = "ip_forward",
                .data                = &ipv4_devconf.data[
                                        IPV4_DEVCONF_FORWARDING - 1],
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = devinet_sysctl_forward,
                .extra1                = &ipv4_devconf,
                .extra2                = &init_net,
        },
};
#endif

static __net_init int devinet_init_net(struct net *net)
{
#ifdef CONFIG_SYSCTL
        struct ctl_table_header *forw_hdr;
        struct ctl_table *tbl;
#endif
        struct ipv4_devconf *all, *dflt;
        int err;
        int i;

        err = -ENOMEM;
        net->ipv4.inet_addr_lst = kmalloc_objs(struct hlist_head,
                                               IN4_ADDR_HSIZE);
        if (!net->ipv4.inet_addr_lst)
                goto err_alloc_hash;

        all = kmemdup(&ipv4_devconf, sizeof(ipv4_devconf), GFP_KERNEL);
        if (!all)
                goto err_alloc_all;

        dflt = kmemdup(&ipv4_devconf_dflt, sizeof(ipv4_devconf_dflt), GFP_KERNEL);
        if (!dflt)
                goto err_alloc_dflt;

#ifdef CONFIG_SYSCTL
        tbl = kmemdup(ctl_forward_entry, sizeof(ctl_forward_entry), GFP_KERNEL);
        if (!tbl)
                goto err_alloc_ctl;

        tbl[0].data = &all->data[IPV4_DEVCONF_FORWARDING - 1];
        tbl[0].extra1 = all;
        tbl[0].extra2 = net;
#endif

        if (!net_eq(net, &init_net)) {
                switch (net_inherit_devconf()) {
                case 3:
                        /* copy from the current netns */
                        memcpy(all, current->nsproxy->net_ns->ipv4.devconf_all,
                               sizeof(ipv4_devconf));
                        memcpy(dflt,
                               current->nsproxy->net_ns->ipv4.devconf_dflt,
                               sizeof(ipv4_devconf_dflt));
                        break;
                case 0:
                case 1:
                        /* copy from init_net */
                        memcpy(all, init_net.ipv4.devconf_all,
                               sizeof(ipv4_devconf));
                        memcpy(dflt, init_net.ipv4.devconf_dflt,
                               sizeof(ipv4_devconf_dflt));
                        break;
                case 2:
                        /* use compiled values */
                        break;
                }
        }

#ifdef CONFIG_SYSCTL
        err = __devinet_sysctl_register(net, "all", NETCONFA_IFINDEX_ALL, all);
        if (err < 0)
                goto err_reg_all;

        err = __devinet_sysctl_register(net, "default",
                                        NETCONFA_IFINDEX_DEFAULT, dflt);
        if (err < 0)
                goto err_reg_dflt;

        err = -ENOMEM;
        forw_hdr = register_net_sysctl_sz(net, "net/ipv4", tbl,
                                          ARRAY_SIZE(ctl_forward_entry));
        if (!forw_hdr)
                goto err_reg_ctl;
        net->ipv4.forw_hdr = forw_hdr;
#endif

        for (i = 0; i < IN4_ADDR_HSIZE; i++)
                INIT_HLIST_HEAD(&net->ipv4.inet_addr_lst[i]);

        INIT_DEFERRABLE_WORK(&net->ipv4.addr_chk_work, check_lifetime);

        net->ipv4.devconf_all = all;
        net->ipv4.devconf_dflt = dflt;
        return 0;

#ifdef CONFIG_SYSCTL
err_reg_ctl:
        __devinet_sysctl_unregister(net, dflt, NETCONFA_IFINDEX_DEFAULT);
err_reg_dflt:
        __devinet_sysctl_unregister(net, all, NETCONFA_IFINDEX_ALL);
err_reg_all:
        kfree(tbl);
err_alloc_ctl:
#endif
        kfree(dflt);
err_alloc_dflt:
        kfree(all);
err_alloc_all:
        kfree(net->ipv4.inet_addr_lst);
err_alloc_hash:
        return err;
}

static __net_exit void devinet_exit_net(struct net *net)
{
#ifdef CONFIG_SYSCTL
        const struct ctl_table *tbl;
#endif

        cancel_delayed_work_sync(&net->ipv4.addr_chk_work);

#ifdef CONFIG_SYSCTL
        tbl = net->ipv4.forw_hdr->ctl_table_arg;
        unregister_net_sysctl_table(net->ipv4.forw_hdr);
        __devinet_sysctl_unregister(net, net->ipv4.devconf_dflt,
                                    NETCONFA_IFINDEX_DEFAULT);
        __devinet_sysctl_unregister(net, net->ipv4.devconf_all,
                                    NETCONFA_IFINDEX_ALL);
        kfree(tbl);
#endif
        kfree(net->ipv4.devconf_dflt);
        kfree(net->ipv4.devconf_all);
        kfree(net->ipv4.inet_addr_lst);
}

static __net_initdata struct pernet_operations devinet_ops = {
        .init = devinet_init_net,
        .exit = devinet_exit_net,
};

static struct rtnl_af_ops inet_af_ops __read_mostly = {
        .family                  = AF_INET,
        .fill_link_af          = inet_fill_link_af,
        .get_link_af_size = inet_get_link_af_size,
        .validate_link_af = inet_validate_link_af,
        .set_link_af          = inet_set_link_af,
};

static const struct rtnl_msg_handler devinet_rtnl_msg_handlers[] __initconst = {
        {.protocol = PF_INET, .msgtype = RTM_NEWADDR, .doit = inet_rtm_newaddr,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.protocol = PF_INET, .msgtype = RTM_DELADDR, .doit = inet_rtm_deladdr,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.protocol = PF_INET, .msgtype = RTM_GETADDR, .dumpit = inet_dump_ifaddr,
         .flags = RTNL_FLAG_DUMP_UNLOCKED | RTNL_FLAG_DUMP_SPLIT_NLM_DONE},
        {.protocol = PF_INET, .msgtype = RTM_GETNETCONF,
         .doit = inet_netconf_get_devconf, .dumpit = inet_netconf_dump_devconf,
         .flags = RTNL_FLAG_DOIT_UNLOCKED | RTNL_FLAG_DUMP_UNLOCKED},
        {.owner = THIS_MODULE, .protocol = PF_INET, .msgtype = RTM_GETMULTICAST,
         .dumpit = inet_dump_ifmcaddr, .flags = RTNL_FLAG_DUMP_UNLOCKED},
};

void __init devinet_init(void)
{
        register_pernet_subsys(&devinet_ops);
        register_netdevice_notifier(&ip_netdev_notifier);

        if (rtnl_af_register(&inet_af_ops))
                panic("Unable to register inet_af_ops\n");

        rtnl_register_many(devinet_rtnl_msg_handlers);
}













































































































































































































































  130 







  123 

















   67 





   75 



   89 
  108 








   11 



   45 







   56 
   56 



   45 













   54 






   13 







   41 
   41 
   41 







   41 

   41 


   41 







   72 










   54 














   54 







   54 









   54 






   54 




















































































   39 

   15 








   54 
















   54 
   54 



    6 












   54 


































































































































































































   54 






   54 









   54 

























































  105 




  104 













  105 


  105 




















    2 





   43 






    4 





   59 




   53 










   20 






   16 


    5 









   58 












   16 


   42 
   42 












   58 


   58 

    4 























    6 











   54 












   54 











   54 






   54 








  105 






    3 









    3 




































































































































































































































































































































































































































































































































































































































































































   30 





























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        NET3:        Implementation of the ICMP protocol layer.
 *
 *                Alan Cox, <alan@lxorguk.ukuu.org.uk>
 *
 *        Some of the function names and the icmp unreach table for this
 *        module were derived from [icmp.c 1.0.11 06/02/93] by
 *        Ross Biro, Fred N. van Kempen, Mark Evans, Alan Cox, Gerhard Koerting.
 *        Other than that this module is a complete rewrite.
 *
 *        Fixes:
 *        Clemens Fruhwirth        :        introduce global icmp rate limiting
 *                                        with icmp type masking ability instead
 *                                        of broken per type icmp timeouts.
 *                Mike Shaver        :        RFC1122 checks.
 *                Alan Cox        :        Multicast ping reply as self.
 *                Alan Cox        :        Fix atomicity lockup in ip_build_xmit
 *                                        call.
 *                Alan Cox        :        Added 216,128 byte paths to the MTU
 *                                        code.
 *                Martin Mares        :        RFC1812 checks.
 *                Martin Mares        :        Can be configured to follow redirects
 *                                        if acting as a router _without_ a
 *                                        routing protocol (RFC 1812).
 *                Martin Mares        :        Echo requests may be configured to
 *                                        be ignored (RFC 1812).
 *                Martin Mares        :        Limitation of ICMP error message
 *                                        transmit rate (RFC 1812).
 *                Martin Mares        :        TOS and Precedence set correctly
 *                                        (RFC 1812).
 *                Martin Mares        :        Now copying as much data from the
 *                                        original packet as we can without
 *                                        exceeding 576 bytes (RFC 1812).
 *        Willy Konynenberg        :        Transparent proxying support.
 *                Keith Owens        :        RFC1191 correction for 4.2BSD based
 *                                        path MTU bug.
 *                Thomas Quinot        :        ICMP Dest Unreach codes up to 15 are
 *                                        valid (RFC 1812).
 *                Andi Kleen        :        Check all packet lengths properly
 *                                        and moved all kfree_skb() up to
 *                                        icmp_rcv.
 *                Andi Kleen        :        Move the rate limit bookkeeping
 *                                        into the dest entry and use a token
 *                                        bucket filter (thanks to ANK). Make
 *                                        the rates sysctl configurable.
 *                Yu Tianli        :        Fixed two ugly bugs in icmp_send
 *                                        - IP option length was accounted wrongly
 *                                        - ICMP header length was not accounted
 *                                          at all.
 *              Tristan Greaves :       Added sysctl option to ignore bogus
 *                                      broadcast responses from broken routers.
 *
 * To Fix:
 *
 *        - Should use skb_pull() instead of all the manual checking.
 *          This would also greatly simply some upper layer error handlers. --AK
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/types.h>
#include <linux/jiffies.h>
#include <linux/kernel.h>
#include <linux/fcntl.h>
#include <linux/nospec.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/inetdevice.h>
#include <linux/netdevice.h>
#include <linux/string.h>
#include <linux/netfilter_ipv4.h>
#include <linux/slab.h>
#include <net/flow.h>
#include <net/snmp.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/protocol.h>
#include <net/icmp.h>
#include <net/tcp.h>
#include <net/udp.h>
#include <net/raw.h>
#include <net/ping.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <linux/errno.h>
#include <linux/timer.h>
#include <linux/init.h>
#include <linux/uaccess.h>
#include <net/checksum.h>
#include <net/xfrm.h>
#include <net/inet_common.h>
#include <net/ip_fib.h>
#include <net/l3mdev.h>
#include <net/addrconf.h>
#include <net/inet_dscp.h>
#define CREATE_TRACE_POINTS
#include <trace/events/icmp.h>

/*
 *        Build xmit assembly blocks
 */

struct icmp_bxm {
        struct sk_buff *skb;
        int offset;
        int data_len;

        struct {
                struct icmphdr icmph;
                __be32               times[3];
        } data;
        int head_len;

        /* Must be last as it ends in a flexible-array member. */
        struct ip_options_rcu replyopts;
};

/* An array of errno for error messages from dest unreach. */
/* RFC 1122: 3.2.2.1 States that NET_UNREACH, HOST_UNREACH and SR_FAILED MUST be considered 'transient errs'. */

const struct icmp_err icmp_err_convert[] = {
        {
                .errno = ENETUNREACH,        /* ICMP_NET_UNREACH */
                .fatal = 0,
        },
        {
                .errno = EHOSTUNREACH,        /* ICMP_HOST_UNREACH */
                .fatal = 0,
        },
        {
                .errno = ENOPROTOOPT        /* ICMP_PROT_UNREACH */,
                .fatal = 1,
        },
        {
                .errno = ECONNREFUSED,        /* ICMP_PORT_UNREACH */
                .fatal = 1,
        },
        {
                .errno = EMSGSIZE,        /* ICMP_FRAG_NEEDED */
                .fatal = 0,
        },
        {
                .errno = EOPNOTSUPP,        /* ICMP_SR_FAILED */
                .fatal = 0,
        },
        {
                .errno = ENETUNREACH,        /* ICMP_NET_UNKNOWN */
                .fatal = 1,
        },
        {
                .errno = EHOSTDOWN,        /* ICMP_HOST_UNKNOWN */
                .fatal = 1,
        },
        {
                .errno = ENONET,        /* ICMP_HOST_ISOLATED */
                .fatal = 1,
        },
        {
                .errno = ENETUNREACH,        /* ICMP_NET_ANO        */
                .fatal = 1,
        },
        {
                .errno = EHOSTUNREACH,        /* ICMP_HOST_ANO */
                .fatal = 1,
        },
        {
                .errno = ENETUNREACH,        /* ICMP_NET_UNR_TOS */
                .fatal = 0,
        },
        {
                .errno = EHOSTUNREACH,        /* ICMP_HOST_UNR_TOS */
                .fatal = 0,
        },
        {
                .errno = EHOSTUNREACH,        /* ICMP_PKT_FILTERED */
                .fatal = 1,
        },
        {
                .errno = EHOSTUNREACH,        /* ICMP_PREC_VIOLATION */
                .fatal = 1,
        },
        {
                .errno = EHOSTUNREACH,        /* ICMP_PREC_CUTOFF */
                .fatal = 1,
        },
};
EXPORT_SYMBOL(icmp_err_convert);

/*
 *        ICMP control array. This specifies what to do with each ICMP.
 */

struct icmp_control {
        enum skb_drop_reason (*handler)(struct sk_buff *skb);
        short   error;                /* This ICMP is classed as an error message */
};

static const struct icmp_control icmp_pointers[NR_ICMP_TYPES+1];

static DEFINE_PER_CPU(struct sock *, ipv4_icmp_sk);

/* Called with BH disabled */
static inline struct sock *icmp_xmit_lock(struct net *net)
{
        struct sock *sk;

        sk = this_cpu_read(ipv4_icmp_sk);

        if (unlikely(!spin_trylock(&sk->sk_lock.slock))) {
                /* This can happen if the output path signals a
                 * dst_link_failure() for an outgoing ICMP packet.
                 */
                return NULL;
        }
        sock_net_set(sk, net);
        return sk;
}

static inline void icmp_xmit_unlock(struct sock *sk)
{
        sock_net_set(sk, &init_net);
        spin_unlock(&sk->sk_lock.slock);
}

/**
 * icmp_global_allow - Are we allowed to send one more ICMP message ?
 * @net: network namespace
 *
 * Uses a token bucket to limit our ICMP messages to ~sysctl_icmp_msgs_per_sec.
 * Returns false if we reached the limit and can not send another packet.
 * Works in tandem with icmp_global_consume().
 */
bool icmp_global_allow(struct net *net)
{
        u32 delta, now, oldstamp;
        int incr, new, old;

        /* Note: many cpus could find this condition true.
         * Then later icmp_global_consume() could consume more credits,
         * this is an acceptable race.
         */
        if (atomic_read(&net->ipv4.icmp_global_credit) > 0)
                return true;

        now = jiffies;
        oldstamp = READ_ONCE(net->ipv4.icmp_global_stamp);
        delta = min_t(u32, now - oldstamp, HZ);
        if (delta < HZ / 50)
                return false;

        incr = READ_ONCE(net->ipv4.sysctl_icmp_msgs_per_sec);
        incr = div_u64((u64)incr * delta, HZ);
        if (!incr)
                return false;

        if (cmpxchg(&net->ipv4.icmp_global_stamp, oldstamp, now) == oldstamp) {
                old = atomic_read(&net->ipv4.icmp_global_credit);
                do {
                        new = min(old + incr, READ_ONCE(net->ipv4.sysctl_icmp_msgs_burst));
                } while (!atomic_try_cmpxchg(&net->ipv4.icmp_global_credit, &old, new));
        }
        return true;
}

void icmp_global_consume(struct net *net)
{
        int credits = get_random_u32_below(3);

        /* Note: this might make icmp_global.credit negative. */
        if (credits)
                atomic_sub(credits, &net->ipv4.icmp_global_credit);
}

static bool icmpv4_mask_allow(struct net *net, int type, int code)
{
        if (type > NR_ICMP_TYPES)
                return true;

        /* Don't limit PMTU discovery. */
        if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED)
                return true;

        /* Limit if icmp type is enabled in ratemask. */
        if (!((1 << type) & READ_ONCE(net->ipv4.sysctl_icmp_ratemask)))
                return true;

        return false;
}

static bool icmpv4_global_allow(struct net *net, int type, int code,
                                bool *apply_ratelimit)
{
        if (icmpv4_mask_allow(net, type, code))
                return true;

        if (icmp_global_allow(net)) {
                *apply_ratelimit = true;
                return true;
        }
        __ICMP_INC_STATS(net, ICMP_MIB_RATELIMITGLOBAL);
        return false;
}

/*
 *        Send an ICMP frame.
 */

static bool icmpv4_xrlim_allow(struct net *net, struct rtable *rt,
                               struct flowi4 *fl4, int type, int code,
                               bool apply_ratelimit)
{
        struct dst_entry *dst = &rt->dst;
        struct inet_peer *peer;
        struct net_device *dev;
        int peer_timeout;
        bool rc = true;

        if (!apply_ratelimit)
                return true;

        peer_timeout = READ_ONCE(net->ipv4.sysctl_icmp_ratelimit);
        if (!peer_timeout)
                goto out;

        /* No rate limit on loopback */
        rcu_read_lock();
        dev = dst_dev_rcu(dst);
        if (dev && (dev->flags & IFF_LOOPBACK))
                goto out_unlock;

        peer = inet_getpeer_v4(net->ipv4.peers, fl4->daddr,
                               l3mdev_master_ifindex_rcu(dev));
        rc = inet_peer_xrlim_allow(peer, peer_timeout);

out_unlock:
        rcu_read_unlock();
out:
        if (!rc)
                __ICMP_INC_STATS(net, ICMP_MIB_RATELIMITHOST);
        else
                icmp_global_consume(net);
        return rc;
}

/*
 *        Maintain the counters used in the SNMP statistics for outgoing ICMP
 */
void icmp_out_count(struct net *net, unsigned char type)
{
        ICMPMSGOUT_INC_STATS(net, type);
        ICMP_INC_STATS(net, ICMP_MIB_OUTMSGS);
}

/*
 *        Checksum each fragment, and on the first include the headers and final
 *        checksum.
 */
static int icmp_glue_bits(void *from, char *to, int offset, int len, int odd,
                          struct sk_buff *skb)
{
        DEFINE_RAW_FLEX(struct icmp_bxm, icmp_param, replyopts.opt.__data,
                        IP_OPTIONS_DATA_FIXED_SIZE);
        __wsum csum;

        icmp_param = from;

        csum = skb_copy_and_csum_bits(icmp_param->skb,
                                      icmp_param->offset + offset,
                                      to, len);

        skb->csum = csum_block_add(skb->csum, csum, odd);
        if (icmp_param->data.icmph.type <= NR_ICMP_TYPES &&
            icmp_pointers[array_index_nospec(icmp_param->data.icmph.type,
                                             NR_ICMP_TYPES + 1)].error)
                nf_ct_attach(skb, icmp_param->skb);
        return 0;
}

static void icmp_push_reply(struct sock *sk,
                            struct icmp_bxm *icmp_param,
                            struct flowi4 *fl4,
                            struct ipcm_cookie *ipc, struct rtable **rt)
{
        struct sk_buff *skb;

        if (ip_append_data(sk, fl4, icmp_glue_bits, icmp_param,
                           icmp_param->data_len+icmp_param->head_len,
                           icmp_param->head_len,
                           ipc, rt, MSG_DONTWAIT) < 0) {
                __ICMP_INC_STATS(sock_net(sk), ICMP_MIB_OUTERRORS);
                ip_flush_pending_frames(sk);
        } else if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
                struct icmphdr *icmph = icmp_hdr(skb);
                __wsum csum;
                struct sk_buff *skb1;

                csum = csum_partial_copy_nocheck((void *)&icmp_param->data,
                                                 (char *)icmph,
                                                 icmp_param->head_len);
                skb_queue_walk(&sk->sk_write_queue, skb1) {
                        csum = csum_add(csum, skb1->csum);
                }
                icmph->checksum = csum_fold(csum);
                skb->ip_summed = CHECKSUM_NONE;
                ip_push_pending_frames(sk, fl4);
        }
}

/*
 *        Driving logic for building and sending ICMP messages.
 */

static void icmp_reply(struct icmp_bxm *icmp_param, struct sk_buff *skb)
{
        struct rtable *rt = skb_rtable(skb);
        struct net *net = dev_net_rcu(rt->dst.dev);
        bool apply_ratelimit = false;
        struct ipcm_cookie ipc;
        struct flowi4 fl4;
        struct sock *sk;
        __be32 daddr, saddr;
        u32 mark = IP4_REPLY_MARK(net, skb->mark);
        int type = icmp_param->data.icmph.type;
        int code = icmp_param->data.icmph.code;

        if (ip_options_echo(net, &icmp_param->replyopts.opt, skb))
                return;

        /* Needed by both icmpv4_global_allow and icmp_xmit_lock */
        local_bh_disable();

        /* is global icmp_msgs_per_sec exhausted ? */
        if (!icmpv4_global_allow(net, type, code, &apply_ratelimit))
                goto out_bh_enable;

        sk = icmp_xmit_lock(net);
        if (!sk)
                goto out_bh_enable;

        icmp_param->data.icmph.checksum = 0;

        ipcm_init(&ipc);
        ipc.tos = ip_hdr(skb)->tos;
        ipc.sockc.mark = mark;
        daddr = ipc.addr = ip_hdr(skb)->saddr;
        saddr = fib_compute_spec_dst(skb);

        if (icmp_param->replyopts.opt.optlen) {
                ipc.opt = &icmp_param->replyopts;
                if (ipc.opt->opt.srr)
                        daddr = icmp_param->replyopts.opt.faddr;
        }
        memset(&fl4, 0, sizeof(fl4));
        fl4.daddr = daddr;
        fl4.saddr = saddr;
        fl4.flowi4_mark = mark;
        fl4.flowi4_uid = sock_net_uid(net, NULL);
        fl4.flowi4_dscp = ip4h_dscp(ip_hdr(skb));
        fl4.flowi4_proto = IPPROTO_ICMP;
        fl4.flowi4_oif = l3mdev_master_ifindex(skb->dev);
        security_skb_classify_flow(skb, flowi4_to_flowi_common(&fl4));
        rt = ip_route_output_key(net, &fl4);
        if (IS_ERR(rt))
                goto out_unlock;
        if (icmpv4_xrlim_allow(net, rt, &fl4, type, code, apply_ratelimit))
                icmp_push_reply(sk, icmp_param, &fl4, &ipc, &rt);
        ip_rt_put(rt);
out_unlock:
        icmp_xmit_unlock(sk);
out_bh_enable:
        local_bh_enable();
}

/*
 * The device used for looking up which routing table to use for sending an ICMP
 * error is preferably the source whenever it is set, which should ensure the
 * icmp error can be sent to the source host, else lookup using the routing
 * table of the destination device, else use the main routing table (index 0).
 */
static struct net_device *icmp_get_route_lookup_dev(struct sk_buff *skb)
{
        struct net_device *dev = skb->dev;
        const struct dst_entry *dst;

        if (dev)
                return dev;
        dst = skb_dst(skb);
        return dst ? dst_dev(dst) : NULL;
}

static struct rtable *icmp_route_lookup(struct net *net, struct flowi4 *fl4,
                                        struct sk_buff *skb_in,
                                        const struct iphdr *iph, __be32 saddr,
                                        dscp_t dscp, u32 mark, int type,
                                        int code, struct icmp_bxm *param)
{
        struct net_device *route_lookup_dev;
        struct dst_entry *dst, *dst2;
        struct rtable *rt, *rt2;
        struct flowi4 fl4_dec;
        int err;

        memset(fl4, 0, sizeof(*fl4));
        fl4->daddr = (param->replyopts.opt.srr ?
                      param->replyopts.opt.faddr : iph->saddr);
        fl4->saddr = saddr;
        fl4->flowi4_mark = mark;
        fl4->flowi4_uid = sock_net_uid(net, NULL);
        fl4->flowi4_dscp = dscp;
        fl4->flowi4_proto = IPPROTO_ICMP;
        fl4->fl4_icmp_type = type;
        fl4->fl4_icmp_code = code;
        route_lookup_dev = icmp_get_route_lookup_dev(skb_in);
        fl4->flowi4_oif = l3mdev_master_ifindex(route_lookup_dev);

        security_skb_classify_flow(skb_in, flowi4_to_flowi_common(fl4));
        rt = ip_route_output_key_hash(net, fl4, skb_in);
        if (IS_ERR(rt))
                return rt;

        /* No need to clone since we're just using its address. */
        rt2 = rt;

        dst = xfrm_lookup(net, &rt->dst,
                          flowi4_to_flowi(fl4), NULL, 0);
        rt = dst_rtable(dst);
        if (!IS_ERR(dst)) {
                if (rt != rt2)
                        return rt;
                if (inet_addr_type_dev_table(net, route_lookup_dev,
                                             fl4->daddr) == RTN_LOCAL)
                        return rt;
        } else if (PTR_ERR(dst) == -EPERM) {
                rt = NULL;
        } else {
                return rt;
        }
        err = xfrm_decode_session_reverse(net, skb_in, flowi4_to_flowi(&fl4_dec), AF_INET);
        if (err)
                goto relookup_failed;

        if (inet_addr_type_dev_table(net, route_lookup_dev,
                                     fl4_dec.saddr) == RTN_LOCAL) {
                rt2 = __ip_route_output_key(net, &fl4_dec);
                if (IS_ERR(rt2))
                        err = PTR_ERR(rt2);
        } else {
                struct flowi4 fl4_2 = {};
                unsigned long orefdst;

                fl4_2.daddr = fl4_dec.saddr;
                rt2 = ip_route_output_key(net, &fl4_2);
                if (IS_ERR(rt2)) {
                        err = PTR_ERR(rt2);
                        goto relookup_failed;
                }
                /* Ugh! */
                orefdst = skb_dstref_steal(skb_in);
                err = ip_route_input(skb_in, fl4_dec.daddr, fl4_dec.saddr,
                                     dscp, rt2->dst.dev) ? -EINVAL : 0;

                dst_release(&rt2->dst);
                rt2 = skb_rtable(skb_in);
                /* steal dst entry from skb_in, don't drop refcnt */
                skb_dstref_steal(skb_in);
                skb_dstref_restore(skb_in, orefdst);

                /*
                 * At this point, fl4_dec.daddr should NOT be local (we
                 * checked fl4_dec.saddr above). However, a race condition
                 * may occur if the address is added to the interface
                 * concurrently. In that case, ip_route_input() returns a
                 * LOCAL route with dst.output=ip_rt_bug, which must not
                 * be used for output.
                 */
                if (!err && rt2 && rt2->rt_type == RTN_LOCAL) {
                        net_warn_ratelimited("detected local route for %pI4 during ICMP sending, src %pI4\n",
                                             &fl4_dec.daddr, &fl4_dec.saddr);
                        dst_release(&rt2->dst);
                        err = -EINVAL;
                }
        }

        if (err)
                goto relookup_failed;

        dst2 = xfrm_lookup(net, &rt2->dst, flowi4_to_flowi(&fl4_dec), NULL,
                           XFRM_LOOKUP_ICMP);
        rt2 = dst_rtable(dst2);
        if (!IS_ERR(dst2)) {
                dst_release(&rt->dst);
                rt = rt2;
        } else if (PTR_ERR(dst2) == -EPERM) {
                if (rt)
                        dst_release(&rt->dst);
                return rt2;
        } else {
                err = PTR_ERR(dst2);
                goto relookup_failed;
        }
        return rt;

relookup_failed:
        if (rt)
                return rt;
        return ERR_PTR(err);
}

struct icmp_ext_iio_addr4_subobj {
        __be16 afi;
        __be16 reserved;
        __be32 addr4;
};

static unsigned int icmp_ext_iio_len(void)
{
        return sizeof(struct icmp_extobj_hdr) +
                /* ifIndex */
                sizeof(__be32) +
                /* Interface Address Sub-Object */
                sizeof(struct icmp_ext_iio_addr4_subobj) +
                /* Interface Name Sub-Object. Length must be a multiple of 4
                 * bytes.
                 */
                ALIGN(sizeof(struct icmp_ext_iio_name_subobj), 4) +
                /* MTU */
                sizeof(__be32);
}

static unsigned int icmp_ext_max_len(u8 ext_objs)
{
        unsigned int ext_max_len;

        ext_max_len = sizeof(struct icmp_ext_hdr);

        if (ext_objs & BIT(ICMP_ERR_EXT_IIO_IIF))
                ext_max_len += icmp_ext_iio_len();

        return ext_max_len;
}

static __be32 icmp_ext_iio_addr4_find(const struct net_device *dev)
{
        struct in_device *in_dev;
        struct in_ifaddr *ifa;

        in_dev = __in_dev_get_rcu(dev);
        if (!in_dev)
                return 0;

        /* It is unclear from RFC 5837 which IP address should be chosen, but
         * it makes sense to choose a global unicast address.
         */
        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                if (READ_ONCE(ifa->ifa_flags) & IFA_F_SECONDARY)
                        continue;
                if (ifa->ifa_scope != RT_SCOPE_UNIVERSE ||
                    ipv4_is_multicast(ifa->ifa_address))
                        continue;
                return ifa->ifa_address;
        }

        return 0;
}

static void icmp_ext_iio_iif_append(struct net *net, struct sk_buff *skb,
                                    int iif)
{
        struct icmp_ext_iio_name_subobj *name_subobj;
        struct icmp_extobj_hdr *objh;
        struct net_device *dev;
        __be32 data;

        if (!iif)
                return;

        /* Add the fields in the order specified by RFC 5837. */
        objh = skb_put(skb, sizeof(*objh));
        objh->class_num = ICMP_EXT_OBJ_CLASS_IIO;
        objh->class_type = ICMP_EXT_CTYPE_IIO_ROLE(ICMP_EXT_CTYPE_IIO_ROLE_IIF);

        data = htonl(iif);
        skb_put_data(skb, &data, sizeof(__be32));
        objh->class_type |= ICMP_EXT_CTYPE_IIO_IFINDEX;

        rcu_read_lock();

        dev = dev_get_by_index_rcu(net, iif);
        if (!dev)
                goto out;

        data = icmp_ext_iio_addr4_find(dev);
        if (data) {
                struct icmp_ext_iio_addr4_subobj *addr4_subobj;

                addr4_subobj = skb_put_zero(skb, sizeof(*addr4_subobj));
                addr4_subobj->afi = htons(ICMP_AFI_IP);
                addr4_subobj->addr4 = data;
                objh->class_type |= ICMP_EXT_CTYPE_IIO_IPADDR;
        }

        name_subobj = skb_put_zero(skb, ALIGN(sizeof(*name_subobj), 4));
        name_subobj->len = ALIGN(sizeof(*name_subobj), 4);
        netdev_copy_name(dev, name_subobj->name);
        objh->class_type |= ICMP_EXT_CTYPE_IIO_NAME;

        data = htonl(READ_ONCE(dev->mtu));
        skb_put_data(skb, &data, sizeof(__be32));
        objh->class_type |= ICMP_EXT_CTYPE_IIO_MTU;

out:
        rcu_read_unlock();
        objh->length = htons(skb_tail_pointer(skb) - (unsigned char *)objh);
}

static void icmp_ext_objs_append(struct net *net, struct sk_buff *skb,
                                 u8 ext_objs, int iif)
{
        if (ext_objs & BIT(ICMP_ERR_EXT_IIO_IIF))
                icmp_ext_iio_iif_append(net, skb, iif);
}

static struct sk_buff *
icmp_ext_append(struct net *net, struct sk_buff *skb_in, struct icmphdr *icmph,
                unsigned int room, int iif)
{
        unsigned int payload_len, ext_max_len, ext_len;
        struct icmp_ext_hdr *ext_hdr;
        struct sk_buff *skb;
        u8 ext_objs;
        int nhoff;

        switch (icmph->type) {
        case ICMP_DEST_UNREACH:
        case ICMP_TIME_EXCEEDED:
        case ICMP_PARAMETERPROB:
                break;
        default:
                return NULL;
        }

        ext_objs = READ_ONCE(net->ipv4.sysctl_icmp_errors_extension_mask);
        if (!ext_objs)
                return NULL;

        ext_max_len = icmp_ext_max_len(ext_objs);
        if (ICMP_EXT_ORIG_DGRAM_MIN_LEN + ext_max_len > room)
                return NULL;

        skb = skb_clone(skb_in, GFP_ATOMIC);
        if (!skb)
                return NULL;

        nhoff = skb_network_offset(skb);
        payload_len = min(skb->len - nhoff, ICMP_EXT_ORIG_DGRAM_MIN_LEN);

        if (!pskb_network_may_pull(skb, payload_len))
                goto free_skb;

        if (pskb_trim(skb, nhoff + ICMP_EXT_ORIG_DGRAM_MIN_LEN) ||
            __skb_put_padto(skb, nhoff + ICMP_EXT_ORIG_DGRAM_MIN_LEN, false))
                goto free_skb;

        if (pskb_expand_head(skb, 0, ext_max_len, GFP_ATOMIC))
                goto free_skb;

        ext_hdr = skb_put_zero(skb, sizeof(*ext_hdr));
        ext_hdr->version = ICMP_EXT_VERSION_2;

        icmp_ext_objs_append(net, skb, ext_objs, iif);

        /* Do not send an empty extension structure. */
        ext_len = skb_tail_pointer(skb) - (unsigned char *)ext_hdr;
        if (ext_len == sizeof(*ext_hdr))
                goto free_skb;

        ext_hdr->checksum = ip_compute_csum(ext_hdr, ext_len);
        /* The length of the original datagram in 32-bit words (RFC 4884). */
        icmph->un.reserved[1] = ICMP_EXT_ORIG_DGRAM_MIN_LEN / sizeof(u32);

        return skb;

free_skb:
        consume_skb(skb);
        return NULL;
}

/*
 *        Send an ICMP message in response to a situation
 *
 *        RFC 1122: 3.2.2        MUST send at least the IP header and 8 bytes of header.
 *                  MAY send more (we do).
 *                        MUST NOT change this header information.
 *                        MUST NOT reply to a multicast/broadcast IP address.
 *                        MUST NOT reply to a multicast/broadcast MAC address.
 *                        MUST reply to only the first fragment.
 */

void __icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info,
                 const struct inet_skb_parm *parm)
{
        DEFINE_RAW_FLEX(struct icmp_bxm, icmp_param, replyopts.opt.__data,
                        IP_OPTIONS_DATA_FIXED_SIZE);
        struct iphdr *iph;
        int room;
        struct rtable *rt = skb_rtable(skb_in);
        bool apply_ratelimit = false;
        struct sk_buff *ext_skb;
        struct ipcm_cookie ipc;
        struct flowi4 fl4;
        __be32 saddr;
        u8  tos;
        u32 mark;
        struct net *net;
        struct sock *sk;

        if (!rt)
                return;

        rcu_read_lock();

        if (rt->dst.dev)
                net = dev_net_rcu(rt->dst.dev);
        else if (skb_in->dev)
                net = dev_net_rcu(skb_in->dev);
        else
                goto out;

        /*
         *        Find the original header. It is expected to be valid, of course.
         *        Check this, icmp_send is called from the most obscure devices
         *        sometimes.
         */
        iph = ip_hdr(skb_in);

        if ((u8 *)iph < skb_in->head ||
            (skb_network_header(skb_in) + sizeof(*iph)) >
            skb_tail_pointer(skb_in))
                goto out;

        /*
         *        No replies to physical multicast/broadcast
         */
        if (skb_in->pkt_type != PACKET_HOST)
                goto out;

        /*
         *        Now check at the protocol level
         */
        if (rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
                goto out;

        /*
         *        Only reply to fragment 0. We byte re-order the constant
         *        mask for efficiency.
         */
        if (iph->frag_off & htons(IP_OFFSET))
                goto out;

        /*
         *        If we send an ICMP error to an ICMP error a mess would result..
         */
        if (icmp_pointers[type].error) {
                /*
                 *        We are an error, check if we are replying to an
                 *        ICMP error
                 */
                if (iph->protocol == IPPROTO_ICMP) {
                        u8 _inner_type, *itp;

                        itp = skb_header_pointer(skb_in,
                                                 skb_network_header(skb_in) +
                                                 (iph->ihl << 2) +
                                                 offsetof(struct icmphdr,
                                                          type) -
                                                 skb_in->data,
                                                 sizeof(_inner_type),
                                                 &_inner_type);
                        if (!itp)
                                goto out;

                        /*
                         *        Assume any unknown ICMP type is an error. This
                         *        isn't specified by the RFC, but think about it..
                         */
                        if (*itp > NR_ICMP_TYPES ||
                            icmp_pointers[*itp].error)
                                goto out;
                }
        }

        /* Needed by both icmpv4_global_allow and icmp_xmit_lock */
        local_bh_disable();

        /* Check global sysctl_icmp_msgs_per_sec ratelimit, unless
         * incoming dev is loopback.  If outgoing dev change to not be
         * loopback, then peer ratelimit still work (in icmpv4_xrlim_allow)
         */
        if (!(skb_in->dev && (skb_in->dev->flags&IFF_LOOPBACK)) &&
              !icmpv4_global_allow(net, type, code, &apply_ratelimit))
                goto out_bh_enable;

        sk = icmp_xmit_lock(net);
        if (!sk)
                goto out_bh_enable;

        /*
         *        Construct source address and options.
         */

        saddr = iph->daddr;
        if (!(rt->rt_flags & RTCF_LOCAL)) {
                struct net_device *dev = NULL;

                rcu_read_lock();
                if (rt_is_input_route(rt) &&
                    READ_ONCE(net->ipv4.sysctl_icmp_errors_use_inbound_ifaddr))
                        dev = dev_get_by_index_rcu(net, parm->iif ? parm->iif :
                                                   inet_iif(skb_in));

                if (dev)
                        saddr = inet_select_addr(dev, iph->saddr,
                                                 RT_SCOPE_LINK);
                else
                        saddr = 0;
                rcu_read_unlock();
        }

        tos = icmp_pointers[type].error ? (RT_TOS(iph->tos) |
                                           IPTOS_PREC_INTERNETCONTROL) :
                                           iph->tos;
        mark = IP4_REPLY_MARK(net, skb_in->mark);

        if (__ip_options_echo(net, &icmp_param->replyopts.opt, skb_in,
                              &parm->opt))
                goto out_unlock;


        /*
         *        Prepare data for ICMP header.
         */

        icmp_param->data.icmph.type         = type;
        icmp_param->data.icmph.code         = code;
        icmp_param->data.icmph.un.gateway = info;
        icmp_param->data.icmph.checksum         = 0;
        icmp_param->skb          = skb_in;
        icmp_param->offset = skb_network_offset(skb_in);
        ipcm_init(&ipc);
        ipc.tos = tos;
        ipc.addr = iph->saddr;
        ipc.opt = &icmp_param->replyopts;
        ipc.sockc.mark = mark;

        rt = icmp_route_lookup(net, &fl4, skb_in, iph, saddr,
                               inet_dsfield_to_dscp(tos), mark, type, code,
                               icmp_param);
        if (IS_ERR(rt))
                goto out_unlock;

        if (rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
                goto ende;

        /* peer icmp_ratelimit */
        if (!icmpv4_xrlim_allow(net, rt, &fl4, type, code, apply_ratelimit))
                goto ende;

        /* RFC says return as much as we can without exceeding 576 bytes. */

        room = dst4_mtu(&rt->dst);
        if (room > 576)
                room = 576;
        room -= sizeof(struct iphdr) + icmp_param->replyopts.opt.optlen;
        room -= sizeof(struct icmphdr);
        /* Guard against tiny mtu. We need to include at least one
         * IP network header for this message to make any sense.
         */
        if (room <= (int)sizeof(struct iphdr))
                goto ende;

        ext_skb = icmp_ext_append(net, skb_in, &icmp_param->data.icmph, room,
                                  parm->iif);
        if (ext_skb)
                icmp_param->skb = ext_skb;

        icmp_param->data_len = icmp_param->skb->len - icmp_param->offset;
        if (icmp_param->data_len > room)
                icmp_param->data_len = room;
        icmp_param->head_len = sizeof(struct icmphdr);

        /* if we don't have a source address at this point, fall back to the
         * dummy address instead of sending out a packet with a source address
         * of 0.0.0.0
         */
        if (!fl4.saddr)
                fl4.saddr = htonl(INADDR_DUMMY);

        trace_icmp_send(skb_in, type, code);

        icmp_push_reply(sk, icmp_param, &fl4, &ipc, &rt);

        if (ext_skb)
                consume_skb(ext_skb);
ende:
        ip_rt_put(rt);
out_unlock:
        icmp_xmit_unlock(sk);
out_bh_enable:
        local_bh_enable();
out:
        rcu_read_unlock();
}
EXPORT_SYMBOL(__icmp_send);

#if IS_ENABLED(CONFIG_NF_NAT)
#include <net/netfilter/nf_conntrack.h>
void icmp_ndo_send(struct sk_buff *skb_in, int type, int code, __be32 info)
{
        struct sk_buff *cloned_skb = NULL;
        enum ip_conntrack_info ctinfo;
        enum ip_conntrack_dir dir;
        struct inet_skb_parm parm;
        struct nf_conn *ct;
        __be32 orig_ip;

        memset(&parm, 0, sizeof(parm));
        ct = nf_ct_get(skb_in, &ctinfo);
        if (!ct || !(READ_ONCE(ct->status) & IPS_NAT_MASK)) {
                __icmp_send(skb_in, type, code, info, &parm);
                return;
        }

        if (skb_shared(skb_in))
                skb_in = cloned_skb = skb_clone(skb_in, GFP_ATOMIC);

        if (unlikely(!skb_in || skb_network_header(skb_in) < skb_in->head ||
            (skb_network_header(skb_in) + sizeof(struct iphdr)) >
            skb_tail_pointer(skb_in) || skb_ensure_writable(skb_in,
            skb_network_offset(skb_in) + sizeof(struct iphdr))))
                goto out;

        orig_ip = ip_hdr(skb_in)->saddr;
        dir = CTINFO2DIR(ctinfo);
        ip_hdr(skb_in)->saddr = ct->tuplehash[dir].tuple.src.u3.ip;
        __icmp_send(skb_in, type, code, info, &parm);
        ip_hdr(skb_in)->saddr = orig_ip;
out:
        consume_skb(cloned_skb);
}
EXPORT_SYMBOL(icmp_ndo_send);
#endif

static void icmp_socket_deliver(struct sk_buff *skb, u32 info)
{
        const struct iphdr *iph = (const struct iphdr *)skb->data;
        const struct net_protocol *ipprot;
        int protocol = iph->protocol;

        /* Checkin full IP header plus 8 bytes of protocol to
         * avoid additional coding at protocol handlers.
         */
        if (!pskb_may_pull(skb, iph->ihl * 4 + 8))
                goto out;

        /* IPPROTO_RAW sockets are not supposed to receive anything. */
        if (protocol == IPPROTO_RAW)
                goto out;

        raw_icmp_error(skb, protocol, info);

        ipprot = rcu_dereference(inet_protos[protocol]);
        if (ipprot && ipprot->err_handler)
                ipprot->err_handler(skb, info);
        return;

out:
        __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS);
}

static bool icmp_tag_validation(int proto)
{
        const struct net_protocol *ipprot;
        bool ok;

        rcu_read_lock();
        ipprot = rcu_dereference(inet_protos[proto]);
        ok = ipprot ? ipprot->icmp_strict_tag_validation : false;
        rcu_read_unlock();
        return ok;
}

/*
 *        Handle ICMP_DEST_UNREACH, ICMP_TIME_EXCEEDED, ICMP_QUENCH, and
 *        ICMP_PARAMETERPROB.
 */

static enum skb_drop_reason icmp_unreach(struct sk_buff *skb)
{
        enum skb_drop_reason reason = SKB_NOT_DROPPED_YET;
        const struct iphdr *iph;
        struct icmphdr *icmph;
        struct net *net;
        u32 info = 0;

        net = skb_dst_dev_net_rcu(skb);

        /*
         *        Incomplete header ?
         *         Only checks for the IP header, there should be an
         *        additional check for longer headers in upper levels.
         */

        if (!pskb_may_pull(skb, sizeof(struct iphdr)))
                goto out_err;

        icmph = icmp_hdr(skb);
        iph   = (const struct iphdr *)skb->data;

        if (iph->ihl < 5)  { /* Mangled header, drop. */
                reason = SKB_DROP_REASON_IP_INHDR;
                goto out_err;
        }

        switch (icmph->type) {
        case ICMP_DEST_UNREACH:
                switch (icmph->code & 15) {
                case ICMP_NET_UNREACH:
                case ICMP_HOST_UNREACH:
                case ICMP_PROT_UNREACH:
                case ICMP_PORT_UNREACH:
                        break;
                case ICMP_FRAG_NEEDED:
                        /* for documentation of the ip_no_pmtu_disc
                         * values please see
                         * Documentation/networking/ip-sysctl.rst
                         */
                        switch (READ_ONCE(net->ipv4.sysctl_ip_no_pmtu_disc)) {
                        default:
                                net_dbg_ratelimited("%pI4: fragmentation needed and DF set\n",
                                                    &iph->daddr);
                                break;
                        case 2:
                                goto out;
                        case 3:
                                if (!icmp_tag_validation(iph->protocol))
                                        goto out;
                                fallthrough;
                        case 0:
                                info = ntohs(icmph->un.frag.mtu);
                        }
                        break;
                case ICMP_SR_FAILED:
                        net_dbg_ratelimited("%pI4: Source Route Failed\n",
                                            &iph->daddr);
                        break;
                default:
                        break;
                }
                if (icmph->code > NR_ICMP_UNREACH)
                        goto out;
                break;
        case ICMP_PARAMETERPROB:
                info = ntohl(icmph->un.gateway) >> 24;
                break;
        case ICMP_TIME_EXCEEDED:
                __ICMP_INC_STATS(net, ICMP_MIB_INTIMEEXCDS);
                if (icmph->code == ICMP_EXC_FRAGTIME)
                        goto out;
                break;
        }

        /*
         *        Throw it at our lower layers
         *
         *        RFC 1122: 3.2.2 MUST extract the protocol ID from the passed
         *                  header.
         *        RFC 1122: 3.2.2.1 MUST pass ICMP unreach messages to the
         *                  transport layer.
         *        RFC 1122: 3.2.2.2 MUST pass ICMP time expired messages to
         *                  transport layer.
         */

        /*
         *        Check the other end isn't violating RFC 1122. Some routers send
         *        bogus responses to broadcast frames. If you see this message
         *        first check your netmask matches at both ends, if it does then
         *        get the other vendor to fix their kit.
         */

        if (!READ_ONCE(net->ipv4.sysctl_icmp_ignore_bogus_error_responses) &&
            inet_addr_type_dev_table(net, skb->dev, iph->daddr) == RTN_BROADCAST) {
                net_warn_ratelimited("%pI4 sent an invalid ICMP type %u, code %u error to a broadcast: %pI4 on %s\n",
                                     &ip_hdr(skb)->saddr,
                                     icmph->type, icmph->code,
                                     &iph->daddr, skb->dev->name);
                goto out;
        }

        icmp_socket_deliver(skb, info);

out:
        return reason;
out_err:
        __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
        return reason ?: SKB_DROP_REASON_NOT_SPECIFIED;
}


/*
 *        Handle ICMP_REDIRECT.
 */

static enum skb_drop_reason icmp_redirect(struct sk_buff *skb)
{
        if (skb->len < sizeof(struct iphdr)) {
                __ICMP_INC_STATS(dev_net_rcu(skb->dev), ICMP_MIB_INERRORS);
                return SKB_DROP_REASON_PKT_TOO_SMALL;
        }

        if (!pskb_may_pull(skb, sizeof(struct iphdr))) {
                /* there aught to be a stat */
                return SKB_DROP_REASON_NOMEM;
        }

        icmp_socket_deliver(skb, ntohl(icmp_hdr(skb)->un.gateway));
        return SKB_NOT_DROPPED_YET;
}

/*
 *        Handle ICMP_ECHO ("ping") and ICMP_EXT_ECHO ("PROBE") requests.
 *
 *        RFC 1122: 3.2.2.6 MUST have an echo server that answers ICMP echo
 *                  requests.
 *        RFC 1122: 3.2.2.6 Data received in the ICMP_ECHO request MUST be
 *                  included in the reply.
 *        RFC 1812: 4.3.3.6 SHOULD have a config option for silently ignoring
 *                  echo requests, MUST have default=NOT.
 *        RFC 8335: 8 MUST have a config option to enable/disable ICMP
 *                  Extended Echo Functionality, MUST be disabled by default
 *        See also WRT handling of options once they are done and working.
 */

static enum skb_drop_reason icmp_echo(struct sk_buff *skb)
{
        DEFINE_RAW_FLEX(struct icmp_bxm, icmp_param, replyopts.opt.__data,
                        IP_OPTIONS_DATA_FIXED_SIZE);
        struct net *net;

        net = skb_dst_dev_net_rcu(skb);
        /* should there be an ICMP stat for ignored echos? */
        if (READ_ONCE(net->ipv4.sysctl_icmp_echo_ignore_all))
                return SKB_NOT_DROPPED_YET;

        icmp_param->data.icmph           = *icmp_hdr(skb);
        icmp_param->skb                   = skb;
        icmp_param->offset           = 0;
        icmp_param->data_len           = skb->len;
        icmp_param->head_len           = sizeof(struct icmphdr);

        if (icmp_param->data.icmph.type == ICMP_ECHO)
                icmp_param->data.icmph.type = ICMP_ECHOREPLY;
        else if (!icmp_build_probe(skb, &icmp_param->data.icmph))
                return SKB_NOT_DROPPED_YET;

        icmp_reply(icmp_param, skb);
        return SKB_NOT_DROPPED_YET;
}

/*        Helper for icmp_echo and icmpv6_echo_reply.
 *        Searches for net_device that matches PROBE interface identifier
 *                and builds PROBE reply message in icmphdr.
 *
 *        Returns false if PROBE responses are disabled via sysctl
 */

bool icmp_build_probe(struct sk_buff *skb, struct icmphdr *icmphdr)
{
        struct net *net = dev_net_rcu(skb->dev);
        struct icmp_ext_hdr *ext_hdr, _ext_hdr;
        struct icmp_ext_echo_iio *iio, _iio;
        struct inet6_dev *in6_dev;
        struct in_device *in_dev;
        struct net_device *dev;
        char buff[IFNAMSIZ];
        u16 ident_len;
        u8 status;

        if (!READ_ONCE(net->ipv4.sysctl_icmp_echo_enable_probe))
                return false;

        /* We currently only support probing interfaces on the proxy node
         * Check to ensure L-bit is set
         */
        if (!(ntohs(icmphdr->un.echo.sequence) & 1))
                return false;
        /* Clear status bits in reply message */
        icmphdr->un.echo.sequence &= htons(0xFF00);
        if (icmphdr->type == ICMP_EXT_ECHO)
                icmphdr->type = ICMP_EXT_ECHOREPLY;
        else
                icmphdr->type = ICMPV6_EXT_ECHO_REPLY;
        ext_hdr = skb_header_pointer(skb, 0, sizeof(_ext_hdr), &_ext_hdr);
        /* Size of iio is class_type dependent.
         * Only check header here and assign length based on ctype in the switch statement
         */
        iio = skb_header_pointer(skb, sizeof(_ext_hdr), sizeof(iio->extobj_hdr), &_iio);
        if (!ext_hdr || !iio)
                goto send_mal_query;
        if (ntohs(iio->extobj_hdr.length) <= sizeof(iio->extobj_hdr) ||
            ntohs(iio->extobj_hdr.length) > sizeof(_iio))
                goto send_mal_query;
        ident_len = ntohs(iio->extobj_hdr.length) - sizeof(iio->extobj_hdr);
        iio = skb_header_pointer(skb, sizeof(_ext_hdr),
                                 sizeof(iio->extobj_hdr) + ident_len, &_iio);
        if (!iio)
                goto send_mal_query;

        status = 0;
        dev = NULL;
        switch (iio->extobj_hdr.class_type) {
        case ICMP_EXT_ECHO_CTYPE_NAME:
                if (ident_len >= IFNAMSIZ)
                        goto send_mal_query;
                memset(buff, 0, sizeof(buff));
                memcpy(buff, &iio->ident.name, ident_len);
                dev = dev_get_by_name(net, buff);
                break;
        case ICMP_EXT_ECHO_CTYPE_INDEX:
                if (ident_len != sizeof(iio->ident.ifindex))
                        goto send_mal_query;
                dev = dev_get_by_index(net, ntohl(iio->ident.ifindex));
                break;
        case ICMP_EXT_ECHO_CTYPE_ADDR:
                if (ident_len < sizeof(iio->ident.addr.ctype3_hdr) ||
                    ident_len != sizeof(iio->ident.addr.ctype3_hdr) +
                                 iio->ident.addr.ctype3_hdr.addrlen)
                        goto send_mal_query;
                switch (ntohs(iio->ident.addr.ctype3_hdr.afi)) {
                case ICMP_AFI_IP:
                        if (iio->ident.addr.ctype3_hdr.addrlen != sizeof(struct in_addr))
                                goto send_mal_query;
                        dev = ip_dev_find(net, iio->ident.addr.ip_addr.ipv4_addr);
                        break;
#if IS_ENABLED(CONFIG_IPV6)
                case ICMP_AFI_IP6:
                        if (iio->ident.addr.ctype3_hdr.addrlen != sizeof(struct in6_addr))
                                goto send_mal_query;
                        dev = ipv6_dev_find(net, &iio->ident.addr.ip_addr.ipv6_addr, dev);
                        dev_hold(dev);
                        break;
#endif
                default:
                        goto send_mal_query;
                }
                break;
        default:
                goto send_mal_query;
        }
        if (!dev) {
                icmphdr->code = ICMP_EXT_CODE_NO_IF;
                return true;
        }
        /* Fill bits in reply message */
        if (dev->flags & IFF_UP)
                status |= ICMP_EXT_ECHOREPLY_ACTIVE;

        in_dev = __in_dev_get_rcu(dev);
        if (in_dev && rcu_access_pointer(in_dev->ifa_list))
                status |= ICMP_EXT_ECHOREPLY_IPV4;

        in6_dev = __in6_dev_get(dev);
        if (in6_dev && !list_empty(&in6_dev->addr_list))
                status |= ICMP_EXT_ECHOREPLY_IPV6;

        dev_put(dev);
        icmphdr->un.echo.sequence |= htons(status);
        return true;
send_mal_query:
        icmphdr->code = ICMP_EXT_CODE_MAL_QUERY;
        return true;
}

/*
 *        Handle ICMP Timestamp requests.
 *        RFC 1122: 3.2.2.8 MAY implement ICMP timestamp requests.
 *                  SHOULD be in the kernel for minimum random latency.
 *                  MUST be accurate to a few minutes.
 *                  MUST be updated at least at 15Hz.
 */
static enum skb_drop_reason icmp_timestamp(struct sk_buff *skb)
{
        DEFINE_RAW_FLEX(struct icmp_bxm, icmp_param, replyopts.opt.__data,
                        IP_OPTIONS_DATA_FIXED_SIZE);
        /*
         *        Too short.
         */
        if (skb->len < 4)
                goto out_err;

        /*
         *        Fill in the current time as ms since midnight UT:
         */
        icmp_param->data.times[1] = inet_current_timestamp();
        icmp_param->data.times[2] = icmp_param->data.times[1];

        BUG_ON(skb_copy_bits(skb, 0, &icmp_param->data.times[0], 4));

        icmp_param->data.icmph           = *icmp_hdr(skb);
        icmp_param->data.icmph.type = ICMP_TIMESTAMPREPLY;
        icmp_param->data.icmph.code = 0;
        icmp_param->skb                   = skb;
        icmp_param->offset           = 0;
        icmp_param->data_len           = 0;
        icmp_param->head_len           = sizeof(struct icmphdr) + 12;
        icmp_reply(icmp_param, skb);
        return SKB_NOT_DROPPED_YET;

out_err:
        __ICMP_INC_STATS(skb_dst_dev_net_rcu(skb), ICMP_MIB_INERRORS);
        return SKB_DROP_REASON_PKT_TOO_SMALL;
}

static enum skb_drop_reason icmp_discard(struct sk_buff *skb)
{
        /* pretend it was a success */
        return SKB_NOT_DROPPED_YET;
}

/*
 *        Deal with incoming ICMP packets.
 */
int icmp_rcv(struct sk_buff *skb)
{
        enum skb_drop_reason reason = SKB_DROP_REASON_NOT_SPECIFIED;
        struct rtable *rt = skb_rtable(skb);
        struct net *net = dev_net_rcu(rt->dst.dev);
        struct icmphdr *icmph;

        if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
                struct sec_path *sp = skb_sec_path(skb);
                int nh;

                if (!(sp && sp->xvec[sp->len - 1]->props.flags &
                                 XFRM_STATE_ICMP)) {
                        reason = SKB_DROP_REASON_XFRM_POLICY;
                        goto drop;
                }

                if (!pskb_may_pull(skb, sizeof(*icmph) + sizeof(struct iphdr)))
                        goto drop;

                nh = skb_network_offset(skb);
                skb_set_network_header(skb, sizeof(*icmph));

                if (!xfrm4_policy_check_reverse(NULL, XFRM_POLICY_IN,
                                                skb)) {
                        reason = SKB_DROP_REASON_XFRM_POLICY;
                        goto drop;
                }

                skb_set_network_header(skb, nh);
        }

        __ICMP_INC_STATS(net, ICMP_MIB_INMSGS);

        if (skb_checksum_simple_validate(skb))
                goto csum_error;

        if (!pskb_pull(skb, sizeof(*icmph)))
                goto error;

        icmph = icmp_hdr(skb);

        ICMPMSGIN_INC_STATS(net, icmph->type);

        /* Check for ICMP Extended Echo (PROBE) messages */
        if (icmph->type == ICMP_EXT_ECHO) {
                /* We can't use icmp_pointers[].handler() because it is an array of
                 * size NR_ICMP_TYPES + 1 (19 elements) and PROBE has code 42.
                 */
                reason = icmp_echo(skb);
                goto reason_check;
        }

        /*
         *        Parse the ICMP message
         */

        if (rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
                /*
                 *        RFC 1122: 3.2.2.6 An ICMP_ECHO to broadcast MAY be
                 *          silently ignored (we let user decide with a sysctl).
                 *        RFC 1122: 3.2.2.8 An ICMP_TIMESTAMP MAY be silently
                 *          discarded if to broadcast/multicast.
                 */
                if ((icmph->type == ICMP_ECHO ||
                     icmph->type == ICMP_TIMESTAMP) &&
                    READ_ONCE(net->ipv4.sysctl_icmp_echo_ignore_broadcasts)) {
                        reason = SKB_DROP_REASON_INVALID_PROTO;
                        goto error;
                }
                if (icmph->type != ICMP_ECHO &&
                    icmph->type != ICMP_TIMESTAMP &&
                    icmph->type != ICMP_ADDRESS &&
                    icmph->type != ICMP_ADDRESSREPLY) {
                        reason = SKB_DROP_REASON_INVALID_PROTO;
                        goto error;
                }
        }

        if (icmph->type == ICMP_EXT_ECHOREPLY ||
            icmph->type == ICMP_ECHOREPLY) {
                reason = ping_rcv(skb);
                return reason ? NET_RX_DROP : NET_RX_SUCCESS;
        }

        /*
         *        18 is the highest 'known' ICMP type. Anything else is a mystery
         *
         *        RFC 1122: 3.2.2  Unknown ICMP messages types MUST be silently
         *                  discarded.
         */
        if (icmph->type > NR_ICMP_TYPES) {
                reason = SKB_DROP_REASON_UNHANDLED_PROTO;
                goto error;
        }

        reason = icmp_pointers[icmph->type].handler(skb);
reason_check:
        if (!reason)  {
                consume_skb(skb);
                return NET_RX_SUCCESS;
        }

drop:
        kfree_skb_reason(skb, reason);
        return NET_RX_DROP;
csum_error:
        reason = SKB_DROP_REASON_ICMP_CSUM;
        __ICMP_INC_STATS(net, ICMP_MIB_CSUMERRORS);
error:
        __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
        goto drop;
}

static bool ip_icmp_error_rfc4884_validate(const struct sk_buff *skb, int off)
{
        struct icmp_extobj_hdr *objh, _objh;
        struct icmp_ext_hdr *exth, _exth;
        u16 olen;

        exth = skb_header_pointer(skb, off, sizeof(_exth), &_exth);
        if (!exth)
                return false;
        if (exth->version != 2)
                return true;

        if (exth->checksum &&
            csum_fold(skb_checksum(skb, off, skb->len - off, 0)))
                return false;

        off += sizeof(_exth);
        while (off < skb->len) {
                objh = skb_header_pointer(skb, off, sizeof(_objh), &_objh);
                if (!objh)
                        return false;

                olen = ntohs(objh->length);
                if (olen < sizeof(_objh))
                        return false;

                off += olen;
                if (off > skb->len)
                        return false;
        }

        return true;
}

void ip_icmp_error_rfc4884(const struct sk_buff *skb,
                           struct sock_ee_data_rfc4884 *out,
                           int thlen, int off)
{
        int hlen;

        /* original datagram headers: end of icmph to payload (skb->data) */
        hlen = -skb_transport_offset(skb) - thlen;

        /* per rfc 4884: minimal datagram length of 128 bytes */
        if (off < 128 || off < hlen)
                return;

        /* kernel has stripped headers: return payload offset in bytes */
        off -= hlen;
        if (off + sizeof(struct icmp_ext_hdr) > skb->len)
                return;

        out->len = off;

        if (!ip_icmp_error_rfc4884_validate(skb, off))
                out->flags |= SO_EE_RFC4884_FLAG_INVALID;
}

int icmp_err(struct sk_buff *skb, u32 info)
{
        struct iphdr *iph = (struct iphdr *)skb->data;
        int offset = iph->ihl<<2;
        struct icmphdr *icmph = (struct icmphdr *)(skb->data + offset);
        struct net *net = dev_net_rcu(skb->dev);
        int type = icmp_hdr(skb)->type;
        int code = icmp_hdr(skb)->code;

        /*
         * Use ping_err to handle all icmp errors except those
         * triggered by ICMP_ECHOREPLY which sent from kernel.
         */
        if (icmph->type != ICMP_ECHOREPLY) {
                ping_err(skb, offset, info);
                return 0;
        }

        if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED)
                ipv4_update_pmtu(skb, net, info, 0, IPPROTO_ICMP);
        else if (type == ICMP_REDIRECT)
                ipv4_redirect(skb, net, 0, IPPROTO_ICMP);

        return 0;
}

/*
 *        This table is the definition of how we handle ICMP.
 */
static const struct icmp_control icmp_pointers[NR_ICMP_TYPES + 1] = {
        [ICMP_ECHOREPLY] = {
                .handler = ping_rcv,
        },
        [1] = {
                .handler = icmp_discard,
                .error = 1,
        },
        [2] = {
                .handler = icmp_discard,
                .error = 1,
        },
        [ICMP_DEST_UNREACH] = {
                .handler = icmp_unreach,
                .error = 1,
        },
        [ICMP_SOURCE_QUENCH] = {
                .handler = icmp_unreach,
                .error = 1,
        },
        [ICMP_REDIRECT] = {
                .handler = icmp_redirect,
                .error = 1,
        },
        [6] = {
                .handler = icmp_discard,
                .error = 1,
        },
        [7] = {
                .handler = icmp_discard,
                .error = 1,
        },
        [ICMP_ECHO] = {
                .handler = icmp_echo,
        },
        [9] = {
                .handler = icmp_discard,
                .error = 1,
        },
        [10] = {
                .handler = icmp_discard,
                .error = 1,
        },
        [ICMP_TIME_EXCEEDED] = {
                .handler = icmp_unreach,
                .error = 1,
        },
        [ICMP_PARAMETERPROB] = {
                .handler = icmp_unreach,
                .error = 1,
        },
        [ICMP_TIMESTAMP] = {
                .handler = icmp_timestamp,
        },
        [ICMP_TIMESTAMPREPLY] = {
                .handler = icmp_discard,
        },
        [ICMP_INFO_REQUEST] = {
                .handler = icmp_discard,
        },
        [ICMP_INFO_REPLY] = {
                .handler = icmp_discard,
        },
        [ICMP_ADDRESS] = {
                .handler = icmp_discard,
        },
        [ICMP_ADDRESSREPLY] = {
                .handler = icmp_discard,
        },
};

static int __net_init icmp_sk_init(struct net *net)
{
        /* Control parameters for ECHO replies. */
        net->ipv4.sysctl_icmp_echo_ignore_all = 0;
        net->ipv4.sysctl_icmp_echo_enable_probe = 0;
        net->ipv4.sysctl_icmp_echo_ignore_broadcasts = 1;

        /* Control parameter - ignore bogus broadcast responses? */
        net->ipv4.sysctl_icmp_ignore_bogus_error_responses = 1;

        /*
         *         Configurable global rate limit.
         *
         *        ratelimit defines tokens/packet consumed for dst->rate_token
         *        bucket ratemask defines which icmp types are ratelimited by
         *        setting        it's bit position.
         *
         *        default:
         *        dest unreachable (3), source quench (4),
         *        time exceeded (11), parameter problem (12)
         */

        net->ipv4.sysctl_icmp_ratelimit = 1 * HZ;
        net->ipv4.sysctl_icmp_ratemask = 0x1818;
        net->ipv4.sysctl_icmp_errors_use_inbound_ifaddr = 0;
        net->ipv4.sysctl_icmp_errors_extension_mask = 0;
        net->ipv4.sysctl_icmp_msgs_per_sec = 10000;
        net->ipv4.sysctl_icmp_msgs_burst = 10000;

        return 0;
}

static struct pernet_operations __net_initdata icmp_sk_ops = {
       .init = icmp_sk_init,
};

int __init icmp_init(void)
{
        int err, i;

        for_each_possible_cpu(i) {
                struct sock *sk;

                err = inet_ctl_sock_create(&sk, PF_INET,
                                           SOCK_RAW, IPPROTO_ICMP, &init_net);
                if (err < 0)
                        return err;

                per_cpu(ipv4_icmp_sk, i) = sk;

                /* Enough space for 2 64K ICMP packets, including
                 * sk_buff/skb_shared_info struct overhead.
                 */
                sk->sk_sndbuf =        2 * SKB_TRUESIZE(64 * 1024);

                /*
                 * Speedup sock_wfree()
                 */
                sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
                inet_sk(sk)->pmtudisc = IP_PMTUDISC_DONT;
        }
        return register_pernet_subsys(&icmp_sk_ops);
}















































    8 
















    5 
    5 



























    6 







    4 






    1 



   12 











   13 








   13 



   13 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
/*
 * linux/can/skb.h
 *
 * Definitions for the CAN network socket buffer
 *
 * Copyright (C) 2012 Oliver Hartkopp <socketcan@hartkopp.net>
 *
 */

#ifndef _CAN_SKB_H
#define _CAN_SKB_H

#include <linux/types.h>
#include <linux/skbuff.h>
#include <linux/can.h>
#include <net/can.h>
#include <net/sock.h>

void can_flush_echo_skb(struct net_device *dev);
int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev,
                     unsigned int idx, unsigned int frame_len);
struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx,
                                   unsigned int *len_ptr,
                                   unsigned int *frame_len_ptr);
unsigned int __must_check can_get_echo_skb(struct net_device *dev,
                                           unsigned int idx,
                                           unsigned int *frame_len_ptr);
void can_free_echo_skb(struct net_device *dev, unsigned int idx,
                       unsigned int *frame_len_ptr);
struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf);
struct sk_buff *alloc_canfd_skb(struct net_device *dev,
                                struct canfd_frame **cfd);
struct sk_buff *alloc_canxl_skb(struct net_device *dev,
                                struct canxl_frame **cxl,
                                unsigned int data_len);
struct sk_buff *alloc_can_err_skb(struct net_device *dev,
                                  struct can_frame **cf);
bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb);

static inline struct can_skb_ext *can_skb_ext_add(struct sk_buff *skb)
{
        struct can_skb_ext *csx = skb_ext_add(skb, SKB_EXT_CAN);

        /* skb_ext_add() returns uninitialized space */
        if (csx)
                csx->can_gw_hops = 0;

        return csx;
}

static inline struct can_skb_ext *can_skb_ext_find(struct sk_buff *skb)
{
        return skb_ext_find(skb, SKB_EXT_CAN);
}

static inline void can_skb_set_owner(struct sk_buff *skb, struct sock *sk)
{
        /* If the socket has already been closed by user space, the
         * refcount may already be 0 (and the socket will be freed
         * after the last TX skb has been freed). So only increase
         * socket refcount if the refcount is > 0.
         */
        if (sk && refcount_inc_not_zero(&sk->sk_refcnt)) {
                skb->destructor = sock_efree;
                skb->sk = sk;
        }
}

/*
 * returns an unshared skb owned by the original sock to be echo'ed back
 */
static inline struct sk_buff *can_create_echo_skb(struct sk_buff *skb)
{
        struct sk_buff *nskb;

        nskb = skb_clone(skb, GFP_ATOMIC);
        if (unlikely(!nskb)) {
                kfree_skb(skb);
                return NULL;
        }

        can_skb_set_owner(nskb, skb->sk);
        consume_skb(skb);
        return nskb;
}

static inline bool can_is_can_skb(const struct sk_buff *skb)
{
        struct can_frame *cf = (struct can_frame *)skb->data;

        /* the CAN specific type of skb is identified by its data length */
        return (skb->len == CAN_MTU && cf->len <= CAN_MAX_DLEN);
}

static inline bool can_is_canfd_skb(const struct sk_buff *skb)
{
        struct canfd_frame *cfd = (struct canfd_frame *)skb->data;

        /* the CAN specific type of skb is identified by its data length */
        return (skb->len == CANFD_MTU && cfd->len <= CANFD_MAX_DLEN);
}

static inline bool can_is_canxl_skb(const struct sk_buff *skb)
{
        const struct canxl_frame *cxl = (struct canxl_frame *)skb->data;

        if (skb->len < CANXL_HDR_SIZE + CANXL_MIN_DLEN || skb->len > CANXL_MTU)
                return false;

        /* this also checks valid CAN XL data length boundaries */
        if (skb->len != CANXL_HDR_SIZE + cxl->len)
                return false;

        return cxl->flags & CANXL_XLF;
}

/* get length element value from can[|fd|xl]_frame structure */
static inline unsigned int can_skb_get_len_val(struct sk_buff *skb)
{
        const struct canxl_frame *cxl = (struct canxl_frame *)skb->data;
        const struct canfd_frame *cfd = (struct canfd_frame *)skb->data;

        if (can_is_canxl_skb(skb))
                return cxl->len;

        return cfd->len;
}

/* get needed data length inside CAN frame for all frame types (RTR aware) */
static inline unsigned int can_skb_get_data_len(struct sk_buff *skb)
{
        unsigned int len = can_skb_get_len_val(skb);
        const struct can_frame *cf = (struct can_frame *)skb->data;

        /* RTR frames have an actual length of zero */
        if (can_is_can_skb(skb) && cf->can_id & CAN_RTR_FLAG)
                return 0;

        return len;
}

#endif /* !_CAN_SKB_H */




























   14 








    1 















    8 













   20 



































   20 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
/* SPDX-License-Identifier: GPL-2.0-or-later */

#ifndef _NET_GSO_H
#define _NET_GSO_H

#include <linux/skbuff.h>

/* Keeps track of mac header offset relative to skb->head.
 * It is useful for TSO of Tunneling protocol. e.g. GRE.
 * For non-tunnel skb it points to skb_mac_header() and for
 * tunnel skb it points to outer mac header.
 * Keeps track of level of encapsulation of network headers.
 */
struct skb_gso_cb {
        union {
                int        mac_offset;
                int        data_offset;
        };
        int        encap_level;
        __wsum        csum;
        __u16        csum_start;
};
#define SKB_GSO_CB_OFFSET        32
#define SKB_GSO_CB(skb) ((struct skb_gso_cb *)((skb)->cb + SKB_GSO_CB_OFFSET))

static inline int skb_tnl_header_len(const struct sk_buff *inner_skb)
{
        return (skb_mac_header(inner_skb) - inner_skb->head) -
                SKB_GSO_CB(inner_skb)->mac_offset;
}

static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra)
{
        int new_headroom, headroom;
        int ret;

        headroom = skb_headroom(skb);
        ret = pskb_expand_head(skb, extra, 0, GFP_ATOMIC);
        if (ret)
                return ret;

        new_headroom = skb_headroom(skb);
        SKB_GSO_CB(skb)->mac_offset += (new_headroom - headroom);
        return 0;
}

static inline void gso_reset_checksum(struct sk_buff *skb, __wsum res)
{
        /* Do not update partial checksums if remote checksum is enabled. */
        if (skb->remcsum_offload)
                return;

        SKB_GSO_CB(skb)->csum = res;
        SKB_GSO_CB(skb)->csum_start = skb_checksum_start(skb) - skb->head;
}

/* Compute the checksum for a gso segment. First compute the checksum value
 * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and
 * then add in skb->csum (checksum from csum_start to end of packet).
 * skb->csum and csum_start are then updated to reflect the checksum of the
 * resultant packet starting from the transport header-- the resultant checksum
 * is in the res argument (i.e. normally zero or ~ of checksum of a pseudo
 * header.
 */
static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res)
{
        unsigned char *csum_start = skb_transport_header(skb);
        int plen = (skb->head + SKB_GSO_CB(skb)->csum_start) - csum_start;
        __wsum partial = SKB_GSO_CB(skb)->csum;

        SKB_GSO_CB(skb)->csum = res;
        SKB_GSO_CB(skb)->csum_start = csum_start - skb->head;

        return csum_fold(csum_partial(csum_start, plen, partial));
}

struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
                                  netdev_features_t features, bool tx_path);

static inline struct sk_buff *skb_gso_segment(struct sk_buff *skb,
                                              netdev_features_t features)
{
        return __skb_gso_segment(skb, features, true);
}

struct sk_buff *skb_eth_gso_segment(struct sk_buff *skb,
                                    netdev_features_t features, __be16 type);

struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
                                    netdev_features_t features);

bool skb_gso_validate_network_len(const struct sk_buff *skb, unsigned int mtu);

bool skb_gso_validate_mac_len(const struct sk_buff *skb, unsigned int len);

static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol,
                                        int pulled_hlen, u16 mac_offset,
                                        int mac_len)
{
        skb->protocol = protocol;
        skb->encapsulation = 1;
        skb_push(skb, pulled_hlen);
        skb_reset_transport_header(skb);
        skb->mac_header = mac_offset;
        skb->network_header = skb->mac_header + mac_len;
        skb->mac_len = mac_len;
}

#endif /* _NET_GSO_H */
































   27 


   27 
   27 
   27 





   13 


   13 
   13 
   13 





   18 


   18 
   18 
   18 





   18 


   18 
   18 
   18 





   18 


   18 
   18 
   18 





   18 


   18 
   18 
   18 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
/* Copyright (C) 2019 Netronome Systems, Inc. */

#ifndef __NET_TC_MPLS_H
#define __NET_TC_MPLS_H

#include <linux/tc_act/tc_mpls.h>
#include <net/act_api.h>

struct tcf_mpls_params {
        int tcfm_action;
        u32 tcfm_label;
        int action; /* tcf_action */
        u8 tcfm_tc;
        u8 tcfm_ttl;
        u8 tcfm_bos;
        __be16 tcfm_proto;
        struct rcu_head        rcu;
};

#define ACT_MPLS_TC_NOT_SET        0xff
#define ACT_MPLS_BOS_NOT_SET        0xff
#define ACT_MPLS_LABEL_NOT_SET        0xffffffff

struct tcf_mpls {
        struct tc_action common;
        struct tcf_mpls_params __rcu *mpls_p;
};
#define to_mpls(a) ((struct tcf_mpls *)a)

static inline u32 tcf_mpls_action(const struct tc_action *a)
{
        u32 tcfm_action;

        rcu_read_lock();
        tcfm_action = rcu_dereference(to_mpls(a)->mpls_p)->tcfm_action;
        rcu_read_unlock();

        return tcfm_action;
}

static inline __be16 tcf_mpls_proto(const struct tc_action *a)
{
        __be16 tcfm_proto;

        rcu_read_lock();
        tcfm_proto = rcu_dereference(to_mpls(a)->mpls_p)->tcfm_proto;
        rcu_read_unlock();

        return tcfm_proto;
}

static inline u32 tcf_mpls_label(const struct tc_action *a)
{
        u32 tcfm_label;

        rcu_read_lock();
        tcfm_label = rcu_dereference(to_mpls(a)->mpls_p)->tcfm_label;
        rcu_read_unlock();

        return tcfm_label;
}

static inline u8 tcf_mpls_tc(const struct tc_action *a)
{
        u8 tcfm_tc;

        rcu_read_lock();
        tcfm_tc = rcu_dereference(to_mpls(a)->mpls_p)->tcfm_tc;
        rcu_read_unlock();

        return tcfm_tc;
}

static inline u8 tcf_mpls_bos(const struct tc_action *a)
{
        u8 tcfm_bos;

        rcu_read_lock();
        tcfm_bos = rcu_dereference(to_mpls(a)->mpls_p)->tcfm_bos;
        rcu_read_unlock();

        return tcfm_bos;
}

static inline u8 tcf_mpls_ttl(const struct tc_action *a)
{
        u8 tcfm_ttl;

        rcu_read_lock();
        tcfm_ttl = rcu_dereference(to_mpls(a)->mpls_p)->tcfm_ttl;
        rcu_read_unlock();

        return tcfm_ttl;
}

#endif /* __NET_TC_MPLS_H */


















































































  521 


  521 








  199 
  334 



  521 







    1 


















   58 












   31 


   31 













 2536 





 2534 

 2537 
 2543 







 2532 


 2258 










  284 
  206 









  153 
  153 















  288 




  288 




























   10 










   77 
   76 














   63 

    3 














   55 














   63 


   63 


   55 





   86 
   86 





















  238 















































    5 










    5 





    2 













    2 
























































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Ethernet-type device handling.
 *
 * Version:        @(#)eth.c        1.0.7        05/25/93
 *
 * Authors:        Ross Biro
 *                Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *                Mark Evans, <evansmp@uhura.aston.ac.uk>
 *                Florian  La Roche, <rzsfl@rz.uni-sb.de>
 *                Alan Cox, <gw4pts@gw4pts.ampr.org>
 *
 * Fixes:
 *                Mr Linux        : Arp problems
 *                Alan Cox        : Generic queue tidyup (very tiny here)
 *                Alan Cox        : eth_header ntohs should be htons
 *                Alan Cox        : eth_rebuild_header missing an htons and
 *                                  minor other things.
 *                Tegge                : Arp bug fixes.
 *                Florian                : Removed many unnecessary functions, code cleanup
 *                                  and changes for new arp and skbuff.
 *                Alan Cox        : Redid header building to reflect new format.
 *                Alan Cox        : ARP only when compiled with CONFIG_INET
 *                Greg Page        : 802.2 and SNAP stuff.
 *                Alan Cox        : MAC layer pointers/new format.
 *                Paul Gortmaker        : eth_copy_and_sum shouldn't csum padding.
 *                Alan Cox        : Protect against forwarding explosions with
 *                                  older network drivers and IFF_ALLMULTI.
 *        Christer Weinigel        : Better rebuild header message.
 *             Andrew Morton    : 26Feb01: kill ether_setup() - use netdev_boot_setup().
 */
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/ip.h>
#include <linux/netdevice.h>
#include <linux/nvmem-consumer.h>
#include <linux/etherdevice.h>
#include <linux/skbuff.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/if_ether.h>
#include <linux/of_net.h>
#include <linux/pci.h>
#include <linux/property.h>
#include <net/dst.h>
#include <net/arp.h>
#include <net/sock.h>
#include <net/ipv6.h>
#include <net/ip.h>
#include <net/dsa.h>
#include <net/flow_dissector.h>
#include <net/gro.h>
#include <linux/uaccess.h>
#include <net/pkt_sched.h>

/**
 * eth_header - create the Ethernet header
 * @skb:        buffer to alter
 * @dev:        source device
 * @type:        Ethernet type field
 * @daddr: destination address (NULL leave destination address)
 * @saddr: source address (NULL use device source address)
 * @len:   packet length (<= skb->len)
 *
 *
 * Set the protocol type. For a packet of type ETH_P_802_3/2 we put the length
 * in here instead.
 */
int eth_header(struct sk_buff *skb, struct net_device *dev,
               unsigned short type,
               const void *daddr, const void *saddr, unsigned int len)
{
        struct ethhdr *eth = skb_push(skb, ETH_HLEN);

        if (type != ETH_P_802_3 && type != ETH_P_802_2)
                eth->h_proto = htons(type);
        else
                eth->h_proto = htons(len);

        /*
         *      Set the source hardware address.
         */

        if (!saddr)
                saddr = dev->dev_addr;
        memcpy(eth->h_source, saddr, ETH_ALEN);

        if (daddr) {
                memcpy(eth->h_dest, daddr, ETH_ALEN);
                return ETH_HLEN;
        }

        /*
         *      Anyway, the loopback-device should never use this function...
         */

        if (dev->flags & (IFF_LOOPBACK | IFF_NOARP)) {
                eth_zero_addr(eth->h_dest);
                return ETH_HLEN;
        }

        return -ETH_HLEN;
}
EXPORT_SYMBOL(eth_header);

/**
 * eth_get_headlen - determine the length of header for an ethernet frame
 * @dev: pointer to network device
 * @data: pointer to start of frame
 * @len: total length of frame
 *
 * Make a best effort attempt to pull the length for all of the headers for
 * a given frame in a linear buffer.
 */
u32 eth_get_headlen(const struct net_device *dev, const void *data, u32 len)
{
        const unsigned int flags = FLOW_DISSECTOR_F_PARSE_1ST_FRAG;
        const struct ethhdr *eth = (const struct ethhdr *)data;
        struct flow_keys_basic keys;

        /* this should never happen, but better safe than sorry */
        if (unlikely(len < sizeof(*eth)))
                return len;

        /* parse any remaining L2/L3 headers, check for L4 */
        if (!skb_flow_dissect_flow_keys_basic(dev_net(dev), NULL, &keys, data,
                                              eth->h_proto, sizeof(*eth),
                                              len, flags))
                return max_t(u32, keys.control.thoff, sizeof(*eth));

        /* parse for any L4 headers */
        return min_t(u32, __skb_get_poff(NULL, data, &keys, len), len);
}
EXPORT_SYMBOL(eth_get_headlen);

/**
 * eth_type_trans - determine the packet's protocol ID.
 * @skb: received socket data
 * @dev: receiving network device
 *
 * The rule here is that we
 * assume 802.3 if the type field is short enough to be a length.
 * This is normal practice and works for any 'now in use' protocol.
 */
__be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev)
{
        const unsigned short *sap;
        const struct ethhdr *eth;
        __be16 res;

        skb->dev = dev;
        skb_reset_mac_header(skb);

        eth = eth_skb_pull_mac(skb);
        eth_skb_pkt_type(skb, dev);

        /*
         * Some variants of DSA tagging don't have an ethertype field
         * at all, so we check here whether one of those tagging
         * variants has been configured on the receiving interface,
         * and if so, set skb->protocol without looking at the packet.
         */
        if (unlikely(netdev_uses_dsa(dev)))
                return htons(ETH_P_XDSA);

        if (likely(eth_proto_is_802_3(eth->h_proto)))
                return eth->h_proto;

        /*
         *      This is a magic hack to spot IPX packets. Older Novell breaks
         *      the protocol design and runs IPX over 802.3 without an 802.2 LLC
         *      layer. We look for FFFF which isn't a used 802.2 SSAP/DSAP. This
         *      won't work for fault tolerant netware but does for the rest.
         *        We use skb->dev as temporary storage to not hit
         *        CONFIG_STACKPROTECTOR_STRONG=y costs on some platforms.
         */
        sap = skb_header_pointer(skb, 0, sizeof(*sap), &skb->dev);
        res = (sap && *sap == 0xFFFF) ? htons(ETH_P_802_3) : htons(ETH_P_802_2);

        /* restore skb->dev in case it was mangled by skb_header_pointer(). */
        skb->dev = dev;
        return res;
}
EXPORT_SYMBOL(eth_type_trans);

int eth_header_parse(const struct sk_buff *skb, const struct net_device *dev,
                     unsigned char *haddr)
{
        const struct ethhdr *eth = eth_hdr(skb);

        memcpy(haddr, eth->h_source, ETH_ALEN);
        return ETH_ALEN;
}
EXPORT_SYMBOL(eth_header_parse);

/**
 * eth_header_cache - fill cache entry from neighbour
 * @neigh: source neighbour
 * @hh: destination cache entry
 * @type: Ethernet type field
 *
 * Create an Ethernet header template from the neighbour.
 */
int eth_header_cache(const struct neighbour *neigh, struct hh_cache *hh, __be16 type)
{
        struct ethhdr *eth;
        const struct net_device *dev = neigh->dev;

        eth = (struct ethhdr *)
            (((u8 *) hh->hh_data) + (HH_DATA_OFF(sizeof(*eth))));

        if (type == htons(ETH_P_802_3))
                return -1;

        eth->h_proto = type;
        memcpy(eth->h_source, dev->dev_addr, ETH_ALEN);
        memcpy(eth->h_dest, neigh->ha, ETH_ALEN);

        /* Pairs with READ_ONCE() in neigh_resolve_output(),
         * neigh_hh_output() and neigh_update_hhs().
         */
        smp_store_release(&hh->hh_len, ETH_HLEN);

        return 0;
}
EXPORT_SYMBOL(eth_header_cache);

/**
 * eth_header_cache_update - update cache entry
 * @hh: destination cache entry
 * @dev: network device
 * @haddr: new hardware address
 *
 * Called by Address Resolution module to notify changes in address.
 */
void eth_header_cache_update(struct hh_cache *hh,
                             const struct net_device *dev,
                             const unsigned char *haddr)
{
        memcpy(((u8 *) hh->hh_data) + HH_DATA_OFF(sizeof(struct ethhdr)),
               haddr, ETH_ALEN);
}
EXPORT_SYMBOL(eth_header_cache_update);

/**
 * eth_header_parse_protocol - extract protocol from L2 header
 * @skb: packet to extract protocol from
 */
__be16 eth_header_parse_protocol(const struct sk_buff *skb)
{
        const struct ethhdr *eth = eth_hdr(skb);

        return eth->h_proto;
}
EXPORT_SYMBOL(eth_header_parse_protocol);

/**
 * eth_prepare_mac_addr_change - prepare for mac change
 * @dev: network device
 * @p: socket address
 */
int eth_prepare_mac_addr_change(struct net_device *dev, void *p)
{
        struct sockaddr *addr = p;

        if (!(dev->priv_flags & IFF_LIVE_ADDR_CHANGE) && netif_running(dev))
                return -EBUSY;
        if (!is_valid_ether_addr(addr->sa_data))
                return -EADDRNOTAVAIL;
        return 0;
}
EXPORT_SYMBOL(eth_prepare_mac_addr_change);

/**
 * eth_commit_mac_addr_change - commit mac change
 * @dev: network device
 * @p: socket address
 */
void eth_commit_mac_addr_change(struct net_device *dev, void *p)
{
        struct sockaddr *addr = p;

        eth_hw_addr_set(dev, addr->sa_data);
}
EXPORT_SYMBOL(eth_commit_mac_addr_change);

/**
 * eth_mac_addr - set new Ethernet hardware address
 * @dev: network device
 * @p: socket address
 *
 * Change hardware address of device.
 *
 * This doesn't change hardware matching, so needs to be overridden
 * for most real devices.
 */
int eth_mac_addr(struct net_device *dev, void *p)
{
        int ret;

        ret = eth_prepare_mac_addr_change(dev, p);
        if (ret < 0)
                return ret;
        eth_commit_mac_addr_change(dev, p);
        return 0;
}
EXPORT_SYMBOL(eth_mac_addr);

int eth_validate_addr(struct net_device *dev)
{
        if (!is_valid_ether_addr(dev->dev_addr))
                return -EADDRNOTAVAIL;

        return 0;
}
EXPORT_SYMBOL(eth_validate_addr);

const struct header_ops eth_header_ops ____cacheline_aligned = {
        .create                = eth_header,
        .parse                = eth_header_parse,
        .cache                = eth_header_cache,
        .cache_update        = eth_header_cache_update,
        .parse_protocol        = eth_header_parse_protocol,
};

/**
 * ether_setup - setup Ethernet network device
 * @dev: network device
 *
 * Fill in the fields of the device structure with Ethernet-generic values.
 */
void ether_setup(struct net_device *dev)
{
        dev->header_ops                = &eth_header_ops;
        dev->type                = ARPHRD_ETHER;
        dev->hard_header_len         = ETH_HLEN;
        dev->min_header_len        = ETH_HLEN;
        dev->mtu                = ETH_DATA_LEN;
        dev->min_mtu                = ETH_MIN_MTU;
        dev->max_mtu                = ETH_DATA_LEN;
        dev->addr_len                = ETH_ALEN;
        dev->tx_queue_len        = DEFAULT_TX_QUEUE_LEN;
        dev->flags                = IFF_BROADCAST|IFF_MULTICAST;
        dev->priv_flags                |= IFF_TX_SKB_SHARING;

        eth_broadcast_addr(dev->broadcast);

}
EXPORT_SYMBOL(ether_setup);

/**
 * alloc_etherdev_mqs - Allocates and sets up an Ethernet device
 * @sizeof_priv: Size of additional driver-private structure to be allocated
 *        for this Ethernet device
 * @txqs: The number of TX queues this device has.
 * @rxqs: The number of RX queues this device has.
 *
 * Fill in the fields of the device structure with Ethernet-generic
 * values. Basically does everything except registering the device.
 *
 * Constructs a new net device, complete with a private data area of
 * size (sizeof_priv).  A 32-byte (not bit) alignment is enforced for
 * this private data area.
 */

struct net_device *alloc_etherdev_mqs(int sizeof_priv, unsigned int txqs,
                                      unsigned int rxqs)
{
        return alloc_netdev_mqs(sizeof_priv, "eth%d", NET_NAME_ENUM,
                                ether_setup, txqs, rxqs);
}
EXPORT_SYMBOL(alloc_etherdev_mqs);

ssize_t sysfs_format_mac(char *buf, const unsigned char *addr, int len)
{
        return sysfs_emit(buf, "%*phC\n", len, addr);
}
EXPORT_SYMBOL(sysfs_format_mac);

struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb)
{
        const struct packet_offload *ptype;
        unsigned int hlen, off_eth;
        struct sk_buff *pp = NULL;
        struct ethhdr *eh, *eh2;
        struct sk_buff *p;
        __be16 type;
        int flush = 1;

        off_eth = skb_gro_offset(skb);
        hlen = off_eth + sizeof(*eh);
        eh = skb_gro_header(skb, hlen, off_eth);
        if (unlikely(!eh))
                goto out;

        flush = 0;

        list_for_each_entry(p, head, list) {
                if (!NAPI_GRO_CB(p)->same_flow)
                        continue;

                eh2 = (struct ethhdr *)(p->data + off_eth);
                if (compare_ether_header(eh, eh2)) {
                        NAPI_GRO_CB(p)->same_flow = 0;
                        continue;
                }
        }

        type = eh->h_proto;

        ptype = gro_find_receive_by_type(type);
        if (ptype == NULL) {
                flush = 1;
                goto out;
        }

        skb_gro_pull(skb, sizeof(*eh));
        skb_gro_postpull_rcsum(skb, eh, sizeof(*eh));

        pp = indirect_call_gro_receive_inet(ptype->callbacks.gro_receive,
                                            ipv6_gro_receive, inet_gro_receive,
                                            head, skb);

out:
        skb_gro_flush_final(skb, pp, flush);

        return pp;
}
EXPORT_SYMBOL(eth_gro_receive);

int eth_gro_complete(struct sk_buff *skb, int nhoff)
{
        struct ethhdr *eh = (struct ethhdr *)(skb->data + nhoff);
        __be16 type = eh->h_proto;
        struct packet_offload *ptype;
        int err = -ENOSYS;

        if (skb->encapsulation)
                skb_set_inner_mac_header(skb, nhoff);

        ptype = gro_find_complete_by_type(type);
        if (ptype != NULL)
                err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
                                         ipv6_gro_complete, inet_gro_complete,
                                         skb, nhoff + sizeof(*eh));

        return err;
}
EXPORT_SYMBOL(eth_gro_complete);

static struct packet_offload eth_packet_offload __read_mostly = {
        .type = cpu_to_be16(ETH_P_TEB),
        .priority = 10,
        .callbacks = {
                .gro_receive = eth_gro_receive,
                .gro_complete = eth_gro_complete,
        },
};

static int __init eth_offload_init(void)
{
        dev_add_offload(&eth_packet_offload);

        return 0;
}

fs_initcall(eth_offload_init);

unsigned char * __weak arch_get_platform_mac_address(void)
{
        return NULL;
}

int eth_platform_get_mac_address(struct device *dev, u8 *mac_addr)
{
        unsigned char *addr;
        int ret;

        ret = of_get_mac_address(dev->of_node, mac_addr);
        if (!ret)
                return 0;

        addr = arch_get_platform_mac_address();
        if (!addr)
                return -ENODEV;

        ether_addr_copy(mac_addr, addr);

        return 0;
}
EXPORT_SYMBOL(eth_platform_get_mac_address);

/**
 * platform_get_ethdev_address - Set netdev's MAC address from a given device
 * @dev:        Pointer to the device
 * @netdev:        Pointer to netdev to write the address to
 *
 * Wrapper around eth_platform_get_mac_address() which writes the address
 * directly to netdev->dev_addr.
 */
int platform_get_ethdev_address(struct device *dev, struct net_device *netdev)
{
        u8 addr[ETH_ALEN] __aligned(2);
        int ret;

        ret = eth_platform_get_mac_address(dev, addr);
        if (!ret)
                eth_hw_addr_set(netdev, addr);
        return ret;
}
EXPORT_SYMBOL(platform_get_ethdev_address);

/**
 * nvmem_get_mac_address - Obtain the MAC address from an nvmem cell named
 * 'mac-address' associated with given device.
 *
 * @dev:        Device with which the mac-address cell is associated.
 * @addrbuf:        Buffer to which the MAC address will be copied on success.
 *
 * Returns 0 on success or a negative error number on failure.
 */
int nvmem_get_mac_address(struct device *dev, void *addrbuf)
{
        struct nvmem_cell *cell;
        const void *mac;
        size_t len;

        cell = nvmem_cell_get(dev, "mac-address");
        if (IS_ERR(cell))
                return PTR_ERR(cell);

        mac = nvmem_cell_read(cell, &len);
        nvmem_cell_put(cell);

        if (IS_ERR(mac))
                return PTR_ERR(mac);

        if (len != ETH_ALEN || !is_valid_ether_addr(mac)) {
                kfree(mac);
                return -EINVAL;
        }

        ether_addr_copy(addrbuf, mac);
        kfree(mac);

        return 0;
}

static int fwnode_get_mac_addr(struct fwnode_handle *fwnode,
                               const char *name, char *addr)
{
        int ret;

        ret = fwnode_property_read_u8_array(fwnode, name, addr, ETH_ALEN);
        if (ret)
                return ret;

        if (!is_valid_ether_addr(addr))
                return -EINVAL;
        return 0;
}

/**
 * fwnode_get_mac_address - Get the MAC from the firmware node
 * @fwnode:        Pointer to the firmware node
 * @addr:        Address of buffer to store the MAC in
 *
 * Search the firmware node for the best MAC address to use.  'mac-address' is
 * checked first, because that is supposed to contain to "most recent" MAC
 * address. If that isn't set, then 'local-mac-address' is checked next,
 * because that is the default address.  If that isn't set, then the obsolete
 * 'address' is checked, just in case we're using an old device tree.
 *
 * Note that the 'address' property is supposed to contain a virtual address of
 * the register set, but some DTS files have redefined that property to be the
 * MAC address.
 *
 * All-zero MAC addresses are rejected, because those could be properties that
 * exist in the firmware tables, but were not updated by the firmware.  For
 * example, the DTS could define 'mac-address' and 'local-mac-address', with
 * zero MAC addresses.  Some older U-Boots only initialized 'local-mac-address'.
 * In this case, the real MAC is in 'local-mac-address', and 'mac-address'
 * exists but is all zeros.
 */
int fwnode_get_mac_address(struct fwnode_handle *fwnode, char *addr)
{
        if (!fwnode_get_mac_addr(fwnode, "mac-address", addr) ||
            !fwnode_get_mac_addr(fwnode, "local-mac-address", addr) ||
            !fwnode_get_mac_addr(fwnode, "address", addr))
                return 0;

        return -ENOENT;
}
EXPORT_SYMBOL(fwnode_get_mac_address);

/**
 * device_get_mac_address - Get the MAC for a given device
 * @dev:        Pointer to the device
 * @addr:        Address of buffer to store the MAC in
 */
int device_get_mac_address(struct device *dev, char *addr)
{
        if (!fwnode_get_mac_address(dev_fwnode(dev), addr))
                return 0;

        return nvmem_get_mac_address(dev, addr);
}
EXPORT_SYMBOL(device_get_mac_address);

/**
 * device_get_ethdev_address - Set netdev's MAC address from a given device
 * @dev:        Pointer to the device
 * @netdev:        Pointer to netdev to write the address to
 *
 * Wrapper around device_get_mac_address() which writes the address
 * directly to netdev->dev_addr.
 */
int device_get_ethdev_address(struct device *dev, struct net_device *netdev)
{
        u8 addr[ETH_ALEN];
        int ret;

        ret = device_get_mac_address(dev, addr);
        if (!ret)
                eth_hw_addr_set(netdev, addr);
        return ret;
}
EXPORT_SYMBOL(device_get_ethdev_address);







































































































































































   13 








    1 



    1 




    3 

















    5 
















    3 







   15 








   15 










    5 














    2 









    2 






























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * mpls tunnels        An implementation mpls tunnels using the light weight tunnel
 *                infrastructure
 *
 * Authors:        Roopa Prabhu, <roopa@cumulusnetworks.com>
 */
#include <linux/types.h>
#include <linux/skbuff.h>
#include <linux/net.h>
#include <linux/module.h>
#include <linux/mpls.h>
#include <linux/vmalloc.h>
#include <net/ip.h>
#include <net/dst.h>
#include <net/lwtunnel.h>
#include <net/netevent.h>
#include <net/netns/generic.h>
#include <net/ip6_fib.h>
#include <net/route.h>
#include <net/mpls_iptunnel.h>
#include <linux/mpls_iptunnel.h>
#include "internal.h"

static const struct nla_policy mpls_iptunnel_policy[MPLS_IPTUNNEL_MAX + 1] = {
        [MPLS_IPTUNNEL_DST]        = { .len = sizeof(u32) },
        [MPLS_IPTUNNEL_TTL]        = { .type = NLA_U8 },
};

static unsigned int mpls_encap_size(struct mpls_iptunnel_encap *en)
{
        /* The size of the layer 2.5 labels to be added for this route */
        return en->labels * sizeof(struct mpls_shim_hdr);
}

static int mpls_xmit(struct sk_buff *skb)
{
        struct mpls_iptunnel_encap *tun_encap_info;
        struct mpls_shim_hdr *hdr;
        struct net_device *out_dev;
        unsigned int hh_len;
        unsigned int new_header_size;
        unsigned int mtu;
        struct dst_entry *dst = skb_dst(skb);
        struct rtable *rt = NULL;
        struct rt6_info *rt6 = NULL;
        struct mpls_dev *out_mdev;
        struct net *net;
        int err = 0;
        bool bos;
        int i;
        unsigned int ttl;

        /* Find the output device */
        out_dev = dst->dev;
        net = dev_net_rcu(out_dev);

        if (!mpls_output_possible(out_dev) ||
            !dst->lwtstate || skb_warn_if_lro(skb))
                goto drop;

        skb_forward_csum(skb);

        tun_encap_info = mpls_lwtunnel_encap(dst->lwtstate);

        /* Obtain the ttl using the following set of rules.
         *
         * LWT ttl propagation setting:
         *  - disabled => use default TTL value from LWT
         *  - enabled  => use TTL value from IPv4/IPv6 header
         *  - default  =>
         *   Global ttl propagation setting:
         *    - disabled => use default TTL value from global setting
         *    - enabled => use TTL value from IPv4/IPv6 header
         */
        if (dst->ops->family == AF_INET) {
                if (tun_encap_info->ttl_propagate == MPLS_TTL_PROP_DISABLED)
                        ttl = tun_encap_info->default_ttl;
                else if (tun_encap_info->ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
                         !net->mpls.ip_ttl_propagate)
                        ttl = net->mpls.default_ttl;
                else
                        ttl = ip_hdr(skb)->ttl;
                rt = dst_rtable(dst);
        } else if (dst->ops->family == AF_INET6) {
                if (tun_encap_info->ttl_propagate == MPLS_TTL_PROP_DISABLED)
                        ttl = tun_encap_info->default_ttl;
                else if (tun_encap_info->ttl_propagate == MPLS_TTL_PROP_DEFAULT &&
                         !net->mpls.ip_ttl_propagate)
                        ttl = net->mpls.default_ttl;
                else
                        ttl = ipv6_hdr(skb)->hop_limit;
                rt6 = dst_rt6_info(dst);
        } else {
                goto drop;
        }

        /* Verify the destination can hold the packet */
        new_header_size = mpls_encap_size(tun_encap_info);
        mtu = mpls_dev_mtu(out_dev);
        if (mpls_pkt_too_big(skb, mtu - new_header_size))
                goto drop;

        hh_len = LL_RESERVED_SPACE(out_dev);
        if (!out_dev->header_ops)
                hh_len = 0;

        /* Ensure there is enough space for the headers in the skb */
        if (skb_cow_head(skb, hh_len + new_header_size))
                goto drop;

        skb_set_inner_protocol(skb, skb->protocol);
        skb_reset_inner_network_header(skb);

        skb_push(skb, new_header_size);

        skb_reset_network_header(skb);

        skb->dev = out_dev;
        skb->protocol = htons(ETH_P_MPLS_UC);

        /* Push the new labels */
        hdr = mpls_hdr(skb);
        bos = true;
        for (i = tun_encap_info->labels - 1; i >= 0; i--) {
                hdr[i] = mpls_entry_encode(tun_encap_info->label[i],
                                           ttl, 0, bos);
                bos = false;
        }

        mpls_stats_inc_outucastpkts(net, out_dev, skb);

        if (rt) {
                if (rt->rt_gw_family == AF_INET6)
                        err = neigh_xmit(NEIGH_ND_TABLE, out_dev, &rt->rt_gw6,
                                         skb);
                else
                        err = neigh_xmit(NEIGH_ARP_TABLE, out_dev, &rt->rt_gw4,
                                         skb);
        } else if (rt6) {
                if (ipv6_addr_v4mapped(&rt6->rt6i_gateway)) {
                        /* 6PE (RFC 4798) */
                        err = neigh_xmit(NEIGH_ARP_TABLE, out_dev, &rt6->rt6i_gateway.s6_addr32[3],
                                         skb);
                } else
                        err = neigh_xmit(NEIGH_ND_TABLE, out_dev, &rt6->rt6i_gateway,
                                         skb);
        }
        if (err)
                net_dbg_ratelimited("%s: packet transmission failed: %d\n",
                                    __func__, err);

        return LWTUNNEL_XMIT_DONE;

drop:
        out_mdev = out_dev ? mpls_dev_rcu(out_dev) : NULL;
        if (out_mdev)
                MPLS_INC_STATS(out_mdev, tx_errors);
        kfree_skb(skb);
        return -EINVAL;
}

static int mpls_build_state(struct net *net, struct nlattr *nla,
                            unsigned int family, const void *cfg,
                            struct lwtunnel_state **ts,
                            struct netlink_ext_ack *extack)
{
        struct mpls_iptunnel_encap *tun_encap_info;
        struct nlattr *tb[MPLS_IPTUNNEL_MAX + 1];
        struct lwtunnel_state *newts;
        u8 n_labels;
        int ret;

        ret = nla_parse_nested_deprecated(tb, MPLS_IPTUNNEL_MAX, nla,
                                          mpls_iptunnel_policy, extack);
        if (ret < 0)
                return ret;

        if (!tb[MPLS_IPTUNNEL_DST]) {
                NL_SET_ERR_MSG(extack, "MPLS_IPTUNNEL_DST attribute is missing");
                return -EINVAL;
        }

        /* determine number of labels */
        if (nla_get_labels(tb[MPLS_IPTUNNEL_DST], MAX_NEW_LABELS,
                           &n_labels, NULL, extack))
                return -EINVAL;

        newts = lwtunnel_state_alloc(struct_size(tun_encap_info, label,
                                                 n_labels));
        if (!newts)
                return -ENOMEM;

        tun_encap_info = mpls_lwtunnel_encap(newts);
        ret = nla_get_labels(tb[MPLS_IPTUNNEL_DST], n_labels,
                             &tun_encap_info->labels, tun_encap_info->label,
                             extack);
        if (ret)
                goto errout;

        tun_encap_info->ttl_propagate = MPLS_TTL_PROP_DEFAULT;

        if (tb[MPLS_IPTUNNEL_TTL]) {
                tun_encap_info->default_ttl = nla_get_u8(tb[MPLS_IPTUNNEL_TTL]);
                /* TTL 0 implies propagate from IP header */
                tun_encap_info->ttl_propagate = tun_encap_info->default_ttl ?
                        MPLS_TTL_PROP_DISABLED :
                        MPLS_TTL_PROP_ENABLED;
        }

        newts->type = LWTUNNEL_ENCAP_MPLS;
        newts->flags |= LWTUNNEL_STATE_XMIT_REDIRECT;
        newts->headroom = mpls_encap_size(tun_encap_info);

        *ts = newts;

        return 0;

errout:
        kfree(newts);
        *ts = NULL;

        return ret;
}

static int mpls_fill_encap_info(struct sk_buff *skb,
                                struct lwtunnel_state *lwtstate)
{
        struct mpls_iptunnel_encap *tun_encap_info;

        tun_encap_info = mpls_lwtunnel_encap(lwtstate);

        if (nla_put_labels(skb, MPLS_IPTUNNEL_DST, tun_encap_info->labels,
                           tun_encap_info->label))
                goto nla_put_failure;

        if (tun_encap_info->ttl_propagate != MPLS_TTL_PROP_DEFAULT &&
            nla_put_u8(skb, MPLS_IPTUNNEL_TTL, tun_encap_info->default_ttl))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int mpls_encap_nlsize(struct lwtunnel_state *lwtstate)
{
        struct mpls_iptunnel_encap *tun_encap_info;
        int nlsize;

        tun_encap_info = mpls_lwtunnel_encap(lwtstate);

        nlsize = nla_total_size(tun_encap_info->labels * 4);

        if (tun_encap_info->ttl_propagate != MPLS_TTL_PROP_DEFAULT)
                nlsize += nla_total_size(1);

        return nlsize;
}

static int mpls_encap_cmp(struct lwtunnel_state *a, struct lwtunnel_state *b)
{
        struct mpls_iptunnel_encap *a_hdr = mpls_lwtunnel_encap(a);
        struct mpls_iptunnel_encap *b_hdr = mpls_lwtunnel_encap(b);
        int l;

        if (a_hdr->labels != b_hdr->labels ||
            a_hdr->ttl_propagate != b_hdr->ttl_propagate ||
            a_hdr->default_ttl != b_hdr->default_ttl)
                return 1;

        for (l = 0; l < a_hdr->labels; l++)
                if (a_hdr->label[l] != b_hdr->label[l])
                        return 1;
        return 0;
}

static const struct lwtunnel_encap_ops mpls_iptun_ops = {
        .build_state = mpls_build_state,
        .xmit = mpls_xmit,
        .fill_encap = mpls_fill_encap_info,
        .get_encap_size = mpls_encap_nlsize,
        .cmp_encap = mpls_encap_cmp,
        .owner = THIS_MODULE,
};

static int __init mpls_iptunnel_init(void)
{
        return lwtunnel_encap_add_ops(&mpls_iptun_ops, LWTUNNEL_ENCAP_MPLS);
}
module_init(mpls_iptunnel_init);

static void __exit mpls_iptunnel_exit(void)
{
        lwtunnel_encap_del_ops(&mpls_iptun_ops, LWTUNNEL_ENCAP_MPLS);
}
module_exit(mpls_iptunnel_exit);

MODULE_ALIAS_RTNL_LWT(MPLS);
MODULE_SOFTDEP("post: mpls_gso");
MODULE_DESCRIPTION("MultiProtocol Label Switching IP Tunnels");
MODULE_LICENSE("GPL v2");








































  447 



































































































































































































































































































































































































    8 


    8 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Generic address resolution entity
 *
 *        Authors:
 *        net_random Alan Cox
 *        net_ratelimit Andi Kleen
 *        in{4,6}_pton YOSHIFUJI Hideaki, Copyright (C)2006 USAGI/WIDE Project
 *
 *        Created by Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
 */

#include <linux/module.h>
#include <linux/hex.h>
#include <linux/jiffies.h>
#include <linux/kernel.h>
#include <linux/ctype.h>
#include <linux/inet.h>
#include <linux/mm.h>
#include <linux/net.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/percpu.h>
#include <linux/init.h>
#include <linux/ratelimit.h>
#include <linux/socket.h>

#include <net/sock.h>
#include <net/net_ratelimit.h>
#include <net/ipv6.h>

#include <asm/byteorder.h>
#include <linux/uaccess.h>

DEFINE_RATELIMIT_STATE(net_ratelimit_state, 5 * HZ, 10);
/*
 * All net warning printk()s should be guarded by this function.
 */
int net_ratelimit(void)
{
        return __ratelimit(&net_ratelimit_state);
}
EXPORT_SYMBOL(net_ratelimit);

/*
 * Convert an ASCII string to binary IP.
 * This is outside of net/ipv4/ because various code that uses IP addresses
 * is otherwise not dependent on the TCP/IP stack.
 */

__be32 in_aton(const char *str)
{
        unsigned int l;
        unsigned int val;
        int i;

        l = 0;
        for (i = 0; i < 4; i++)        {
                l <<= 8;
                if (*str != '\0') {
                        val = 0;
                        while (*str != '\0' && *str != '.' && *str != '\n') {
                                val *= 10;
                                val += *str - '0';
                                str++;
                        }
                        l |= val;
                        if (*str != '\0')
                                str++;
                }
        }
        return htonl(l);
}
EXPORT_SYMBOL(in_aton);

#define IN6PTON_XDIGIT                0x00010000
#define IN6PTON_DIGIT                0x00020000
#define IN6PTON_COLON_MASK        0x00700000
#define IN6PTON_COLON_1                0x00100000        /* single : requested */
#define IN6PTON_COLON_2                0x00200000        /* second : requested */
#define IN6PTON_COLON_1_2        0x00400000        /* :: requested */
#define IN6PTON_DOT                0x00800000        /* . */
#define IN6PTON_DELIM                0x10000000
#define IN6PTON_NULL                0x20000000        /* first/tail */
#define IN6PTON_UNKNOWN                0x40000000

static inline int xdigit2bin(char c, int delim)
{
        int val;

        if (c == delim || c == '\0')
                return IN6PTON_DELIM;
        if (c == ':')
                return IN6PTON_COLON_MASK;
        if (c == '.')
                return IN6PTON_DOT;

        val = hex_to_bin(c);
        if (val >= 0)
                return val | IN6PTON_XDIGIT | (val < 10 ? IN6PTON_DIGIT : 0);

        if (delim == -1)
                return IN6PTON_DELIM;
        return IN6PTON_UNKNOWN;
}

/**
 * in4_pton - convert an IPv4 address from literal to binary representation
 * @src: the start of the IPv4 address string
 * @srclen: the length of the string, -1 means strlen(src)
 * @dst: the binary (u8[4] array) representation of the IPv4 address
 * @delim: the delimiter of the IPv4 address in @src, -1 means no delimiter
 * @end: A pointer to the end of the parsed string will be placed here
 *
 * Return one on success, return zero when any error occurs
 * and @end will point to the end of the parsed string.
 *
 */
int in4_pton(const char *src, int srclen,
             u8 *dst,
             int delim, const char **end)
{
        const char *s;
        u8 *d;
        u8 dbuf[4];
        int ret = 0;
        int i;
        int w = 0;

        if (srclen < 0)
                srclen = strlen(src);
        s = src;
        d = dbuf;
        i = 0;
        while (1) {
                int c;
                c = xdigit2bin(srclen > 0 ? *s : '\0', delim);
                if (!(c & (IN6PTON_DIGIT | IN6PTON_DOT | IN6PTON_DELIM | IN6PTON_COLON_MASK))) {
                        goto out;
                }
                if (c & (IN6PTON_DOT | IN6PTON_DELIM | IN6PTON_COLON_MASK)) {
                        if (w == 0)
                                goto out;
                        *d++ = w & 0xff;
                        w = 0;
                        i++;
                        if (c & (IN6PTON_DELIM | IN6PTON_COLON_MASK)) {
                                if (i != 4)
                                        goto out;
                                break;
                        }
                        goto cont;
                }
                w = (w * 10) + c;
                if ((w & 0xffff) > 255) {
                        goto out;
                }
cont:
                if (i >= 4)
                        goto out;
                s++;
                srclen--;
        }
        ret = 1;
        memcpy(dst, dbuf, sizeof(dbuf));
out:
        if (end)
                *end = s;
        return ret;
}
EXPORT_SYMBOL(in4_pton);

/**
 * in6_pton - convert an IPv6 address from literal to binary representation
 * @src: the start of the IPv6 address string
 * @srclen: the length of the string, -1 means strlen(src)
 * @dst: the binary (u8[16] array) representation of the IPv6 address
 * @delim: the delimiter of the IPv6 address in @src, -1 means no delimiter
 * @end: A pointer to the end of the parsed string will be placed here
 *
 * Return one on success, return zero when any error occurs
 * and @end will point to the end of the parsed string.
 *
 */
int in6_pton(const char *src, int srclen,
             u8 *dst,
             int delim, const char **end)
{
        const char *s, *tok = NULL;
        u8 *d, *dc = NULL;
        u8 dbuf[16];
        int ret = 0;
        int i;
        int state = IN6PTON_COLON_1_2 | IN6PTON_XDIGIT | IN6PTON_NULL;
        int w = 0;

        memset(dbuf, 0, sizeof(dbuf));

        s = src;
        d = dbuf;
        if (srclen < 0)
                srclen = strlen(src);

        while (1) {
                int c;

                c = xdigit2bin(srclen > 0 ? *s : '\0', delim);
                if (!(c & state))
                        goto out;
                if (c & (IN6PTON_DELIM | IN6PTON_COLON_MASK)) {
                        /* process one 16-bit word */
                        if (!(state & IN6PTON_NULL)) {
                                *d++ = (w >> 8) & 0xff;
                                *d++ = w & 0xff;
                        }
                        w = 0;
                        if (c & IN6PTON_DELIM) {
                                /* We've processed last word */
                                break;
                        }
                        /*
                         * COLON_1 => XDIGIT
                         * COLON_2 => XDIGIT|DELIM
                         * COLON_1_2 => COLON_2
                         */
                        switch (state & IN6PTON_COLON_MASK) {
                        case IN6PTON_COLON_2:
                                dc = d;
                                state = IN6PTON_XDIGIT | IN6PTON_DELIM;
                                if (dc - dbuf >= sizeof(dbuf))
                                        state |= IN6PTON_NULL;
                                break;
                        case IN6PTON_COLON_1|IN6PTON_COLON_1_2:
                                state = IN6PTON_XDIGIT | IN6PTON_COLON_2;
                                break;
                        case IN6PTON_COLON_1:
                                state = IN6PTON_XDIGIT;
                                break;
                        case IN6PTON_COLON_1_2:
                                state = IN6PTON_COLON_2;
                                break;
                        default:
                                state = 0;
                        }
                        tok = s + 1;
                        goto cont;
                }

                if (c & IN6PTON_DOT) {
                        ret = in4_pton(tok ? tok : s, srclen + (int)(s - tok), d, delim, &s);
                        if (ret > 0) {
                                d += 4;
                                break;
                        }
                        goto out;
                }

                w = (w << 4) | (0xff & c);
                state = IN6PTON_COLON_1 | IN6PTON_DELIM;
                if (!(w & 0xf000)) {
                        state |= IN6PTON_XDIGIT;
                }
                if (!dc && d + 2 < dbuf + sizeof(dbuf)) {
                        state |= IN6PTON_COLON_1_2;
                        state &= ~IN6PTON_DELIM;
                }
                if (d + 2 >= dbuf + sizeof(dbuf)) {
                        state &= ~(IN6PTON_COLON_1|IN6PTON_COLON_1_2);
                }
cont:
                if ((dc && d + 4 < dbuf + sizeof(dbuf)) ||
                    d + 4 == dbuf + sizeof(dbuf)) {
                        state |= IN6PTON_DOT;
                }
                if (d >= dbuf + sizeof(dbuf)) {
                        state &= ~(IN6PTON_XDIGIT|IN6PTON_COLON_MASK);
                }
                s++;
                srclen--;
        }

        i = 15; d--;

        if (dc) {
                while (d >= dc)
                        dst[i--] = *d--;
                while (i >= dc - dbuf)
                        dst[i--] = 0;
                while (i >= 0)
                        dst[i--] = *d--;
        } else
                memcpy(dst, dbuf, sizeof(dbuf));

        ret = 1;
out:
        if (end)
                *end = s;
        return ret;
}
EXPORT_SYMBOL(in6_pton);

static int inet4_pton(const char *src, u16 port_num,
                struct sockaddr_storage *addr)
{
        struct sockaddr_in *addr4 = (struct sockaddr_in *)addr;
        size_t srclen = strlen(src);

        if (srclen > INET_ADDRSTRLEN)
                return -EINVAL;

        if (in4_pton(src, srclen, (u8 *)&addr4->sin_addr.s_addr,
                     '\n', NULL) == 0)
                return -EINVAL;

        addr4->sin_family = AF_INET;
        addr4->sin_port = htons(port_num);

        return 0;
}

static int inet6_pton(struct net *net, const char *src, u16 port_num,
                struct sockaddr_storage *addr)
{
        struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)addr;
        const char *scope_delim;
        size_t srclen = strlen(src);

        if (srclen > INET6_ADDRSTRLEN)
                return -EINVAL;

        if (in6_pton(src, srclen, (u8 *)&addr6->sin6_addr.s6_addr,
                     '%', &scope_delim) == 0)
                return -EINVAL;

        if (ipv6_addr_type(&addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL &&
            src + srclen != scope_delim && *scope_delim == '%') {
                struct net_device *dev;
                char scope_id[16];
                size_t scope_len = min_t(size_t, sizeof(scope_id) - 1,
                                         src + srclen - scope_delim - 1);

                memcpy(scope_id, scope_delim + 1, scope_len);
                scope_id[scope_len] = '\0';

                dev = dev_get_by_name(net, scope_id);
                if (dev) {
                        addr6->sin6_scope_id = dev->ifindex;
                        dev_put(dev);
                } else if (kstrtouint(scope_id, 0, &addr6->sin6_scope_id)) {
                        return -EINVAL;
                }
        }

        addr6->sin6_family = AF_INET6;
        addr6->sin6_port = htons(port_num);

        return 0;
}

/**
 * inet_pton_with_scope - convert an IPv4/IPv6 and port to socket address
 * @net: net namespace (used for scope handling)
 * @af: address family, AF_INET, AF_INET6 or AF_UNSPEC for either
 * @src: the start of the address string
 * @port: the start of the port string (or NULL for none)
 * @addr: output socket address
 *
 * Return zero on success, return errno when any error occurs.
 */
int inet_pton_with_scope(struct net *net, __kernel_sa_family_t af,
                const char *src, const char *port, struct sockaddr_storage *addr)
{
        u16 port_num;
        int ret = -EINVAL;

        if (port) {
                if (kstrtou16(port, 0, &port_num))
                        return -EINVAL;
        } else {
                port_num = 0;
        }

        switch (af) {
        case AF_INET:
                ret = inet4_pton(src, port_num, addr);
                break;
        case AF_INET6:
                ret = inet6_pton(net, src, port_num, addr);
                break;
        case AF_UNSPEC:
                ret = inet4_pton(src, port_num, addr);
                if (ret)
                        ret = inet6_pton(net, src, port_num, addr);
                break;
        default:
                pr_err("unexpected address family %d\n", af);
        }

        return ret;
}
EXPORT_SYMBOL(inet_pton_with_scope);

bool inet_addr_is_any(struct sockaddr_storage *addr)
{
        if (addr->ss_family == AF_INET6) {
                struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)addr;
                const struct sockaddr_in6 in6_any =
                        { .sin6_addr = IN6ADDR_ANY_INIT };

                if (!memcmp(in6->sin6_addr.s6_addr,
                            in6_any.sin6_addr.s6_addr, 16))
                        return true;
        } else if (addr->ss_family == AF_INET) {
                struct sockaddr_in *in = (struct sockaddr_in *)addr;

                if (in->sin_addr.s_addr == htonl(INADDR_ANY))
                        return true;
        } else {
                pr_warn("unexpected address family %u\n", addr->ss_family);
        }

        return false;
}
EXPORT_SYMBOL(inet_addr_is_any);

void inet_proto_csum_replace4(__sum16 *sum, struct sk_buff *skb,
                              __be32 from, __be32 to, bool pseudohdr)
{
        if (skb->ip_summed != CHECKSUM_PARTIAL) {
                csum_replace4(sum, from, to);
                if (skb->ip_summed == CHECKSUM_COMPLETE && pseudohdr)
                        skb->csum = ~csum_add(csum_sub(~(skb->csum),
                                                       (__force __wsum)from),
                                              (__force __wsum)to);
        } else if (pseudohdr)
                *sum = ~csum_fold(csum_add(csum_sub(csum_unfold(*sum),
                                                    (__force __wsum)from),
                                           (__force __wsum)to));
}
EXPORT_SYMBOL(inet_proto_csum_replace4);

/**
 * inet_proto_csum_replace16 - update layer 4 header checksum field
 * @sum: Layer 4 header checksum field
 * @skb: sk_buff for the packet
 * @from: old IPv6 address
 * @to: new IPv6 address
 * @pseudohdr: True if layer 4 header checksum includes pseudoheader
 *
 * Update layer 4 header as per the update in IPv6 src/dst address.
 *
 * There is no need to update skb->csum in this function, because update in two
 * fields a.) IPv6 src/dst address and b.) L4 header checksum cancels each other
 * for skb->csum calculation. Whereas inet_proto_csum_replace4 function needs to
 * update skb->csum, because update in 3 fields a.) IPv4 src/dst address,
 * b.) IPv4 Header checksum and c.) L4 header checksum results in same diff as
 * L4 Header checksum for skb->csum calculation.
 */
void inet_proto_csum_replace16(__sum16 *sum, struct sk_buff *skb,
                               const __be32 *from, const __be32 *to,
                               bool pseudohdr)
{
        __be32 diff[] = {
                ~from[0], ~from[1], ~from[2], ~from[3],
                to[0], to[1], to[2], to[3],
        };
        if (skb->ip_summed != CHECKSUM_PARTIAL) {
                *sum = csum_fold(csum_partial(diff, sizeof(diff),
                                 ~csum_unfold(*sum)));
        } else if (pseudohdr)
                *sum = ~csum_fold(csum_partial(diff, sizeof(diff),
                                  csum_unfold(*sum)));
}
EXPORT_SYMBOL(inet_proto_csum_replace16);

void inet_proto_csum_replace_by_diff(__sum16 *sum, struct sk_buff *skb,
                                     __wsum diff, bool pseudohdr, bool ipv6)
{
        if (skb->ip_summed != CHECKSUM_PARTIAL) {
                csum_replace_by_diff(sum, diff);
                if (skb->ip_summed == CHECKSUM_COMPLETE && pseudohdr && !ipv6)
                        skb->csum = ~csum_sub(diff, skb->csum);
        } else if (pseudohdr) {
                *sum = ~csum_fold(csum_add(diff, csum_unfold(*sum)));
        }
}
EXPORT_SYMBOL(inet_proto_csum_replace_by_diff);













































































 1872 























   22 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * include/linux/folio_batch.h
 *
 * In many places it is efficient to batch an operation up against multiple
 * folios.  A folio_batch is a container which is used for that.
 */

#ifndef _LINUX_FOLIO_BATCH_H
#define _LINUX_FOLIO_BATCH_H

#include <linux/types.h>

/* 31 pointers + header align the folio_batch structure to a power of two */
#define FOLIO_BATCH_SIZE        31

struct folio;

/**
 * struct folio_batch - A collection of folios.
 *
 * The folio_batch is used to amortise the cost of retrieving and
 * operating on a set of folios.  The order of folios in the batch may be
 * significant (eg delete_from_page_cache_batch()).  Some users of the
 * folio_batch store "exceptional" entries in it which can be removed
 * by calling folio_batch_remove_exceptionals().
 */
struct folio_batch {
        unsigned char nr;
        unsigned char i;
        bool percpu_pvec_drained;
        struct folio *folios[FOLIO_BATCH_SIZE];
};

/**
 * folio_batch_init() - Initialise a batch of folios
 * @fbatch: The folio batch.
 *
 * A freshly initialised folio_batch contains zero folios.
 */
static inline void folio_batch_init(struct folio_batch *fbatch)
{
        fbatch->nr = 0;
        fbatch->i = 0;
        fbatch->percpu_pvec_drained = false;
}

static inline void folio_batch_reinit(struct folio_batch *fbatch)
{
        fbatch->nr = 0;
        fbatch->i = 0;
}

static inline unsigned int folio_batch_count(const struct folio_batch *fbatch)
{
        return fbatch->nr;
}

static inline unsigned int folio_batch_space(const struct folio_batch *fbatch)
{
        return FOLIO_BATCH_SIZE - fbatch->nr;
}

/**
 * folio_batch_add() - Add a folio to a batch.
 * @fbatch: The folio batch.
 * @folio: The folio to add.
 *
 * The folio is added to the end of the batch.
 * The batch must have previously been initialised using folio_batch_init().
 *
 * Return: The number of slots still available.
 */
static inline unsigned folio_batch_add(struct folio_batch *fbatch,
                struct folio *folio)
{
        fbatch->folios[fbatch->nr++] = folio;
        return folio_batch_space(fbatch);
}

/**
 * folio_batch_next - Return the next folio to process.
 * @fbatch: The folio batch being processed.
 *
 * Use this function to implement a queue of folios.
 *
 * Return: The next folio in the queue, or NULL if the queue is empty.
 */
static inline struct folio *folio_batch_next(struct folio_batch *fbatch)
{
        if (fbatch->i == fbatch->nr)
                return NULL;
        return fbatch->folios[fbatch->i++];
}

void __folio_batch_release(struct folio_batch *fbatch);

static inline void folio_batch_release(struct folio_batch *fbatch)
{
        if (folio_batch_count(fbatch))
                __folio_batch_release(fbatch);
}

void folio_batch_remove_exceptionals(struct folio_batch *fbatch);
#endif /* _LINUX_FOLIO_BATCH_H */















































































































































































































































    5 


    2 

    2 





    2 



    1 









































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
// SPDX-License-Identifier: GPL-2.0
/*
 *        xt_time
 *        Copyright © CC Computer Consultants GmbH, 2007
 *
 *        based on ipt_time by Fabrice MARIE <fabrice@netfilter.org>
 *        This is a module which is used for time matching
 *        It is using some modified code from dietlibc (localtime() function)
 *        that you can find at https://www.fefe.de/dietlibc/
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/ktime.h>
#include <linux/module.h>
#include <linux/rtc.h>
#include <linux/skbuff.h>
#include <linux/types.h>
#include <linux/netfilter/x_tables.h>
#include <linux/netfilter/xt_time.h>

struct xtm {
        u_int8_t month;    /* (1-12) */
        u_int8_t monthday; /* (1-31) */
        u_int8_t weekday;  /* (1-7) */
        u_int8_t hour;     /* (0-23) */
        u_int8_t minute;   /* (0-59) */
        u_int8_t second;   /* (0-59) */
        unsigned int dse;
};

extern struct timezone sys_tz; /* ouch */

static const u_int16_t days_since_year[] = {
        0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334,
};

static const u_int16_t days_since_leapyear[] = {
        0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335,
};

/*
 * Since time progresses forward, it is best to organize this array in reverse,
 * to minimize lookup time.
 */
enum {
        DSE_FIRST = 2039,
        SECONDS_PER_DAY = 86400,
};
static const u_int16_t days_since_epoch[] = {
        /* 2039 - 2030 */
        25202, 24837, 24472, 24106, 23741, 23376, 23011, 22645, 22280, 21915,
        /* 2029 - 2020 */
        21550, 21184, 20819, 20454, 20089, 19723, 19358, 18993, 18628, 18262,
        /* 2019 - 2010 */
        17897, 17532, 17167, 16801, 16436, 16071, 15706, 15340, 14975, 14610,
        /* 2009 - 2000 */
        14245, 13879, 13514, 13149, 12784, 12418, 12053, 11688, 11323, 10957,
        /* 1999 - 1990 */
        10592, 10227, 9862, 9496, 9131, 8766, 8401, 8035, 7670, 7305,
        /* 1989 - 1980 */
        6940, 6574, 6209, 5844, 5479, 5113, 4748, 4383, 4018, 3652,
        /* 1979 - 1970 */
        3287, 2922, 2557, 2191, 1826, 1461, 1096, 730, 365, 0,
};

/*
 * Each network packet has a (nano)seconds-since-the-epoch (SSTE) timestamp.
 * Since we match against days and daytime, the SSTE value needs to be
 * computed back into human-readable dates.
 *
 * This is done in three separate functions so that the most expensive
 * calculations are done last, in case a "simple match" can be found earlier.
 */
static inline unsigned int localtime_1(struct xtm *r, time64_t time)
{
        unsigned int v, w;

        /* Each day has 86400s, so finding the hour/minute is actually easy. */
        div_u64_rem(time, SECONDS_PER_DAY, &v);
        r->second = v % 60;
        w         = v / 60;
        r->minute = w % 60;
        r->hour   = w / 60;
        return v;
}

static inline void localtime_2(struct xtm *r, time64_t time)
{
        /*
         * Here comes the rest (weekday, monthday). First, divide the SSTE
         * by seconds-per-day to get the number of _days_ since the epoch.
         */
        r->dse = div_u64(time, SECONDS_PER_DAY);

        /*
         * 1970-01-01 (w=0) was a Thursday (4).
         * -1 and +1 map Sunday properly onto 7.
         */
        r->weekday = (4 + r->dse - 1) % 7 + 1;
}

static void localtime_3(struct xtm *r, time64_t time)
{
        unsigned int year, i, w = r->dse;

        /*
         * In each year, a certain number of days-since-the-epoch have passed.
         * Find the year that is closest to said days.
         *
         * Consider, for example, w=21612 (2029-03-04). Loop will abort on
         * dse[i] <= w, which happens when dse[i] == 21550. This implies
         * year == 2009. w will then be 62.
         */
        for (i = 0, year = DSE_FIRST; days_since_epoch[i] > w;
            ++i, --year)
                /* just loop */;

        w -= days_since_epoch[i];

        /*
         * By now we have the current year, and the day of the year.
         * r->yearday = w;
         *
         * On to finding the month (like above). In each month, a certain
         * number of days-since-New Year have passed, and find the closest
         * one.
         *
         * Consider w=62 (in a non-leap year). Loop will abort on
         * dsy[i] < w, which happens when dsy[i] == 31+28 (i == 2).
         * Concludes i == 2, i.e. 3rd month => March.
         *
         * (A different approach to use would be to subtract a monthlength
         * from w repeatedly while counting.)
         */
        if (is_leap_year(year)) {
                /* use days_since_leapyear[] in a leap year */
                for (i = ARRAY_SIZE(days_since_leapyear) - 1;
                    i > 0 && days_since_leapyear[i] > w; --i)
                        /* just loop */;
                r->monthday = w - days_since_leapyear[i] + 1;
        } else {
                for (i = ARRAY_SIZE(days_since_year) - 1;
                    i > 0 && days_since_year[i] > w; --i)
                        /* just loop */;
                r->monthday = w - days_since_year[i] + 1;
        }

        r->month    = i + 1;
}

static bool
time_mt(const struct sk_buff *skb, struct xt_action_param *par)
{
        const struct xt_time_info *info = par->matchinfo;
        unsigned int packet_time;
        struct xtm current_time;
        time64_t stamp;

        /*
         * We need real time here, but we can neither use skb->tstamp
         * nor __net_timestamp().
         *
         * skb->tstamp and skb->skb_mstamp_ns overlap, however, they
         * use different clock types (real vs monotonic).
         *
         * Suppose you have two rules:
         *        1. match before 13:00
         *        2. match after 13:00
         *
         * If you match against processing time (ktime_get_real_seconds) it
         * may happen that the same packet matches both rules if
         * it arrived at the right moment before 13:00, so it would be
         * better to check skb->tstamp and set it via __net_timestamp()
         * if needed.  This however breaks outgoing packets tx timestamp,
         * and causes them to get delayed forever by fq packet scheduler.
         */
        stamp = ktime_get_real_seconds();

        if (info->flags & XT_TIME_LOCAL_TZ)
                /* Adjust for local timezone */
                stamp -= 60 * sys_tz.tz_minuteswest;

        /*
         * xt_time will match when _all_ of the following hold:
         *   - 'now' is in the global time range date_start..date_end
         *   - 'now' is in the monthday mask
         *   - 'now' is in the weekday mask
         *   - 'now' is in the daytime range time_start..time_end
         * (and by default, libxt_time will set these so as to match)
         *
         * note: info->date_start/stop are unsigned 32-bit values that
         *         can hold values beyond y2038, but not after y2106.
         */

        if (stamp < info->date_start || stamp > info->date_stop)
                return false;

        packet_time = localtime_1(&current_time, stamp);

        if (info->daytime_start < info->daytime_stop) {
                if (packet_time < info->daytime_start ||
                    packet_time > info->daytime_stop)
                        return false;
        } else {
                if (packet_time < info->daytime_start &&
                    packet_time > info->daytime_stop)
                        return false;

                /** if user asked to ignore 'next day', then e.g.
                 *  '1 PM Wed, August 1st' should be treated
                 *  like 'Tue 1 PM July 31st'.
                 *
                 * This also causes
                 * 'Monday, "23:00 to 01:00", to match for 2 hours, starting
                 * Monday 23:00 to Tuesday 01:00.
                 */
                if ((info->flags & XT_TIME_CONTIGUOUS) &&
                     packet_time <= info->daytime_stop)
                        stamp -= SECONDS_PER_DAY;
        }

        localtime_2(&current_time, stamp);

        if (!(info->weekdays_match & (1U << current_time.weekday)))
                return false;

        /* Do not spend time computing monthday if all days match anyway */
        if (info->monthdays_match != XT_TIME_ALL_MONTHDAYS) {
                localtime_3(&current_time, stamp);
                if (!(info->monthdays_match & (1U << current_time.monthday)))
                        return false;
        }

        return true;
}

static int time_mt_check(const struct xt_mtchk_param *par)
{
        const struct xt_time_info *info = par->matchinfo;

        if (info->daytime_start > XT_TIME_MAX_DAYTIME ||
            info->daytime_stop > XT_TIME_MAX_DAYTIME) {
                pr_info_ratelimited("invalid argument - start or stop time greater than 23:59:59\n");
                return -EDOM;
        }

        if (info->flags & ~XT_TIME_ALL_FLAGS) {
                pr_info_ratelimited("unknown flags 0x%x\n",
                                    info->flags & ~XT_TIME_ALL_FLAGS);
                return -EINVAL;
        }

        if ((info->flags & XT_TIME_CONTIGUOUS) &&
             info->daytime_start < info->daytime_stop)
                return -EINVAL;

        return 0;
}

static struct xt_match xt_time_mt_reg __read_mostly = {
        .name       = "time",
        .family     = NFPROTO_UNSPEC,
        .match      = time_mt,
        .checkentry = time_mt_check,
        .matchsize  = sizeof(struct xt_time_info),
        .me         = THIS_MODULE,
};

static int __init time_mt_init(void)
{
        int minutes = sys_tz.tz_minuteswest;

        if (minutes < 0) /* east of Greenwich */
                pr_info("kernel timezone is +%02d%02d\n",
                        -minutes / 60, -minutes % 60);
        else /* west of Greenwich */
                pr_info("kernel timezone is -%02d%02d\n",
                        minutes / 60, minutes % 60);

        return xt_register_match(&xt_time_mt_reg);
}

static void __exit time_mt_exit(void)
{
        xt_unregister_match(&xt_time_mt_reg);
}

module_init(time_mt_init);
module_exit(time_mt_exit);
MODULE_AUTHOR("Jan Engelhardt <jengelh@medozas.de>");
MODULE_DESCRIPTION("Xtables: time-based matching");
MODULE_LICENSE("GPL");
MODULE_ALIAS("ipt_time");
MODULE_ALIAS("ip6t_time");

































































































































































































































  480 

























   23 
  457 


















  480 




  479 
  479 

  480 


















  480 

































































    6 






































































































































    6 

















































































  582 











  582 

  322 

  582 

























  581 




  582 

  581 

   15 
  576 



  581 


  582 























    2 




  577 
   12 


































 24005 
 24073 

























   21 


 2623 




   59 
   60 

   59 


   60 


































































































 2905 
 2905 



 2907 









 2626 









    3 


  262 





  150 










   83 








 2615 


































































































































































    1 











































  479 
  478 


  479 

  479 




  459 


   24 







  100 












 3254 



 3253 









 3244 
   12 


 3210 


   99 










   46 





   59 




  477 
 3280 

  473 












 3255 





















   46 



    6 













 7124 



   46 


   18 












 7121 





 3177 




 5919 
    1 










































  479 







 5912 



 3284 






 3282 

















 7227 





 7220 













 7227 



 6908 
  540 


















 7139 


  478 












 2144 


































    1 
   37 









    1 














 7211 











 7218 





 7232 
 2904 

















 3706 










 3713 





 3254 

 3254 


















































































































  431 





  433 












   62 







   62 






















































 1957 





  589 

   24 







  548 


  207 



  349 









    1 






 2499 


























 2503 






 2499 




















 2497 
































 15371 





 15633 

 15252 


 8175 





































































































































































 23983 




 3236 







 23982 
 23981 








  519 


 23998 

 23989 
 23994 
















































































































































    6 





















































    6 




















































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
// SPDX-License-Identifier: GPL-2.0
/*
 *  Copyright(C) 2005-2006, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org>
 *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
 *
 *  High-resolution kernel timers
 *
 *  In contrast to the low-resolution timeout API, aka timer wheel,
 *  hrtimers provide finer resolution and accuracy depending on system
 *  configuration and capabilities.
 *
 *  Started by: Thomas Gleixner and Ingo Molnar
 *
 *  Credits:
 *        Based on the original timer wheel code
 *
 *        Help, testing, suggestions, bugfixes, improvements were
 *        provided by:
 *
 *        George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
 *        et. al.
 */

#include <linux/cpu.h>
#include <linux/export.h>
#include <linux/percpu.h>
#include <linux/hrtimer.h>
#include <linux/notifier.h>
#include <linux/syscalls.h>
#include <linux/interrupt.h>
#include <linux/tick.h>
#include <linux/err.h>
#include <linux/debugobjects.h>
#include <linux/sched/signal.h>
#include <linux/sched/sysctl.h>
#include <linux/sched/rt.h>
#include <linux/sched/deadline.h>
#include <linux/sched/nohz.h>
#include <linux/sched/debug.h>
#include <linux/sched/isolation.h>
#include <linux/timer.h>
#include <linux/freezer.h>
#include <linux/compat.h>

#include <linux/uaccess.h>

#include <trace/events/timer.h>

#include "tick-internal.h"

/*
 * Constants to set the queued state of the timer (INACTIVE, ENQUEUED)
 *
 * The callback state is kept separate in the CPU base because having it in
 * the timer would required touching the timer after the callback, which
 * makes it impossible to free the timer from the callback function.
 *
 * Therefore we track the callback state in:
 *
 *        timer->base->cpu_base->running == timer
 *
 * On SMP it is possible to have a "callback function running and enqueued"
 * status. It happens for example when a posix timer expired and the callback
 * queued a signal. Between dropping the lock which protects the posix timer
 * and reacquiring the base lock of the hrtimer, another CPU can deliver the
 * signal and rearm the timer.
 *
 * All state transitions are protected by cpu_base->lock.
 */
#define HRTIMER_STATE_INACTIVE        false
#define HRTIMER_STATE_ENQUEUED        true

/*
 * The resolution of the clocks. The resolution value is returned in
 * the clock_getres() system call to give application programmers an
 * idea of the (in)accuracy of timers. Timer values are rounded up to
 * this resolution values.
 */
#define HIGH_RES_NSEC                1

/*
 * Masks for selecting the soft and hard context timers from
 * cpu_base->active
 */
#define MASK_SHIFT                (HRTIMER_BASE_MONOTONIC_SOFT)
#define HRTIMER_ACTIVE_HARD        ((1U << MASK_SHIFT) - 1)
#define HRTIMER_ACTIVE_SOFT        (HRTIMER_ACTIVE_HARD << MASK_SHIFT)
#define HRTIMER_ACTIVE_ALL        (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)

static void retrigger_next_event(void *arg);
static ktime_t __hrtimer_cb_get_time(clockid_t clock_id);

/*
 * The timer bases:
 *
 * There are more clockids than hrtimer bases. Thus, we index
 * into the timer bases by the hrtimer_base_type enum. When trying
 * to reach a base using a clockid, hrtimer_clockid_to_base()
 * is used to convert from clockid to the proper hrtimer_base_type.
 */

#define BASE_INIT(idx, cid)                        \
        [idx] = { .index = idx, .clockid = cid }

DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
{
        .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
        .clock_base = {
                BASE_INIT(HRTIMER_BASE_MONOTONIC,        CLOCK_MONOTONIC),
                BASE_INIT(HRTIMER_BASE_REALTIME,        CLOCK_REALTIME),
                BASE_INIT(HRTIMER_BASE_BOOTTIME,        CLOCK_BOOTTIME),
                BASE_INIT(HRTIMER_BASE_TAI,                CLOCK_TAI),
                BASE_INIT(HRTIMER_BASE_MONOTONIC_SOFT,        CLOCK_MONOTONIC),
                BASE_INIT(HRTIMER_BASE_REALTIME_SOFT,        CLOCK_REALTIME),
                BASE_INIT(HRTIMER_BASE_BOOTTIME_SOFT,        CLOCK_BOOTTIME),
                BASE_INIT(HRTIMER_BASE_TAI_SOFT,        CLOCK_TAI),
        },
        .csd = CSD_INIT(retrigger_next_event, NULL)
};

static inline bool hrtimer_base_is_online(struct hrtimer_cpu_base *base)
{
        if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
                return true;
        else
                return likely(base->online);
}

#ifdef CONFIG_HIGH_RES_TIMERS
DEFINE_STATIC_KEY_FALSE(hrtimer_highres_enabled_key);

static void hrtimer_hres_workfn(struct work_struct *work)
{
        static_branch_enable(&hrtimer_highres_enabled_key);
}

static DECLARE_WORK(hrtimer_hres_work, hrtimer_hres_workfn);

static inline void hrtimer_schedule_hres_work(void)
{
        if (!hrtimer_highres_enabled())
                schedule_work(&hrtimer_hres_work);
}
#else
static inline void hrtimer_schedule_hres_work(void) { }
#endif

/*
 * Functions and macros which are different for UP/SMP systems are kept in a
 * single place
 */
#ifdef CONFIG_SMP
/*
 * We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
 * such that hrtimer_callback_running() can unconditionally dereference
 * timer->base->cpu_base
 */
static struct hrtimer_cpu_base migration_cpu_base = {
        .clock_base = {
                [0] = {
                        .cpu_base = &migration_cpu_base,
                        .seq      = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,
                                                             &migration_cpu_base.lock),
                },
        },
};

#define migration_base        migration_cpu_base.clock_base[0]

/*
 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
 * means that all timers which are tied to this base via timer->base are
 * locked, and the base itself is locked too.
 *
 * So __run_timers/migrate_timers can safely modify all timers which could
 * be found on the lists/queues.
 *
 * When the timer's base is locked, and the timer removed from list, it is
 * possible to set timer->base = &migration_base and drop the lock: the timer
 * remains locked.
 */
static struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
                                                    unsigned long *flags)
        __acquires(&timer->base->lock)
{
        for (;;) {
                struct hrtimer_clock_base *base = READ_ONCE(timer->base);

                if (likely(base != &migration_base)) {
                        raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
                        if (likely(base == timer->base))
                                return base;
                        /* The timer has migrated to another CPU: */
                        raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
                }
                cpu_relax();
        }
}

/*
 * Check if the elected target is suitable considering its next
 * event and the hotplug state of the current CPU.
 *
 * If the elected target is remote and its next event is after the timer
 * to queue, then a remote reprogram is necessary. However there is no
 * guarantee the IPI handling the operation would arrive in time to meet
 * the high resolution deadline. In this case the local CPU becomes a
 * preferred target, unless it is offline.
 *
 * High and low resolution modes are handled the same way for simplicity.
 *
 * Called with cpu_base->lock of target cpu held.
 */
static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base,
                                    struct hrtimer_cpu_base *new_cpu_base,
                                    struct hrtimer_cpu_base *this_cpu_base)
{
        ktime_t expires;

        /*
         * The local CPU clockevent can be reprogrammed. Also get_target_base()
         * guarantees it is online.
         */
        if (new_cpu_base == this_cpu_base)
                return true;

        /*
         * The offline local CPU can't be the default target if the
         * next remote target event is after this timer. Keep the
         * elected new base. An IPI will be issued to reprogram
         * it as a last resort.
         */
        if (!hrtimer_base_is_online(this_cpu_base))
                return true;

        expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);

        return expires >= new_base->cpu_base->expires_next;
}

static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, bool pinned)
{
        if (!hrtimer_base_is_online(base)) {
                int cpu = cpumask_any_and(cpu_online_mask, housekeeping_cpumask(HK_TYPE_TIMER));

                return &per_cpu(hrtimer_bases, cpu);
        }

#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
        if (static_branch_likely(&timers_migration_enabled) && !pinned)
                return &per_cpu(hrtimer_bases, get_nohz_timer_target());
#endif
        return base;
}

/*
 * We switch the timer base to a power-optimized selected CPU target,
 * if:
 *        - NO_HZ_COMMON is enabled
 *        - timer migration is enabled
 *        - the timer callback is not running
 *        - the timer is not the first expiring timer on the new target
 *
 * If one of the above requirements is not fulfilled we move the timer
 * to the current CPU or leave it on the previously assigned CPU if
 * the timer callback is currently running.
 */
static inline struct hrtimer_clock_base *
switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, bool pinned)
{
        struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
        struct hrtimer_clock_base *new_base;
        int basenum = base->index;

        this_cpu_base = this_cpu_ptr(&hrtimer_bases);
        new_cpu_base = get_target_base(this_cpu_base, pinned);
again:
        new_base = &new_cpu_base->clock_base[basenum];

        if (base != new_base) {
                /*
                 * We are trying to move timer to new_base. However we can't
                 * change timer's base while it is running, so we keep it on
                 * the same CPU. No hassle vs. reprogramming the event source
                 * in the high resolution case. The remote CPU will take care
                 * of this when the timer function has completed. There is no
                 * conflict as we hold the lock until the timer is enqueued.
                 */
                if (unlikely(hrtimer_callback_running(timer)))
                        return base;

                /* See the comment in lock_hrtimer_base() */
                WRITE_ONCE(timer->base, &migration_base);
                raw_spin_unlock(&base->cpu_base->lock);
                raw_spin_lock(&new_base->cpu_base->lock);

                if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) {
                        raw_spin_unlock(&new_base->cpu_base->lock);
                        raw_spin_lock(&base->cpu_base->lock);
                        new_cpu_base = this_cpu_base;
                        WRITE_ONCE(timer->base, base);
                        goto again;
                }
                WRITE_ONCE(timer->base, new_base);
        } else {
                if (!hrtimer_suitable_target(timer, new_base,  new_cpu_base, this_cpu_base)) {
                        new_cpu_base = this_cpu_base;
                        goto again;
                }
        }
        return new_base;
}

#else /* CONFIG_SMP */

static inline struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
                                                           unsigned long *flags)
        __acquires(&timer->base->cpu_base->lock)
{
        struct hrtimer_clock_base *base = timer->base;

        raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
        return base;
}

# define switch_hrtimer_base(t, b, p)        (b)

#endif        /* !CONFIG_SMP */

/*
 * Functions for the union type storage format of ktime_t which are
 * too large for inlining:
 */
#if BITS_PER_LONG < 64
/*
 * Divide a ktime value by a nanosecond value
 */
s64 __ktime_divns(const ktime_t kt, s64 div)
{
        int sft = 0;
        s64 dclc;
        u64 tmp;

        dclc = ktime_to_ns(kt);
        tmp = dclc < 0 ? -dclc : dclc;

        /* Make sure the divisor is less than 2^32: */
        while (div >> 32) {
                sft++;
                div >>= 1;
        }
        tmp >>= sft;
        do_div(tmp, (u32) div);
        return dclc < 0 ? -tmp : tmp;
}
EXPORT_SYMBOL_GPL(__ktime_divns);
#endif /* BITS_PER_LONG < 64 */

/*
 * Add two ktime values and do a safety check for overflow:
 */
ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
{
        ktime_t res = ktime_add_unsafe(lhs, rhs);

        /*
         * We use KTIME_SEC_MAX here, the maximum timeout which we can
         * return to user space in a timespec:
         */
        if (res < 0 || res < lhs || res < rhs)
                res = ktime_set(KTIME_SEC_MAX, 0);

        return res;
}

EXPORT_SYMBOL_GPL(ktime_add_safe);

#ifdef CONFIG_DEBUG_OBJECTS_TIMERS

static const struct debug_obj_descr hrtimer_debug_descr;

static void *hrtimer_debug_hint(void *addr)
{
        return ACCESS_PRIVATE((struct hrtimer *)addr, function);
}

/*
 * fixup_init is called when:
 * - an active object is initialized
 */
static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
{
        struct hrtimer *timer = addr;

        switch (state) {
        case ODEBUG_STATE_ACTIVE:
                hrtimer_cancel(timer);
                debug_object_init(timer, &hrtimer_debug_descr);
                return true;
        default:
                return false;
        }
}

/*
 * fixup_activate is called when:
 * - an active object is activated
 * - an unknown non-static object is activated
 */
static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
{
        switch (state) {
        case ODEBUG_STATE_ACTIVE:
                WARN_ON(1);
                fallthrough;
        default:
                return false;
        }
}

/*
 * fixup_free is called when:
 * - an active object is freed
 */
static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
{
        struct hrtimer *timer = addr;

        switch (state) {
        case ODEBUG_STATE_ACTIVE:
                hrtimer_cancel(timer);
                debug_object_free(timer, &hrtimer_debug_descr);
                return true;
        default:
                return false;
        }
}

/* Stub timer callback for improperly used timers. */
static enum hrtimer_restart stub_timer(struct hrtimer *unused)
{
        WARN_ON_ONCE(1);
        return HRTIMER_NORESTART;
}

/*
 * hrtimer_fixup_assert_init is called when:
 * - an untracked/uninit-ed object is found
 */
static bool hrtimer_fixup_assert_init(void *addr, enum debug_obj_state state)
{
        struct hrtimer *timer = addr;

        switch (state) {
        case ODEBUG_STATE_NOTAVAILABLE:
                hrtimer_setup(timer, stub_timer, CLOCK_MONOTONIC, 0);
                return true;
        default:
                return false;
        }
}

static const struct debug_obj_descr hrtimer_debug_descr = {
        .name                        = "hrtimer",
        .debug_hint                = hrtimer_debug_hint,
        .fixup_init                = hrtimer_fixup_init,
        .fixup_activate                = hrtimer_fixup_activate,
        .fixup_free                = hrtimer_fixup_free,
        .fixup_assert_init        = hrtimer_fixup_assert_init,
};

static inline void debug_hrtimer_init(struct hrtimer *timer)
{
        debug_object_init(timer, &hrtimer_debug_descr);
}

static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer)
{
        debug_object_init_on_stack(timer, &hrtimer_debug_descr);
}

static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode)
{
        debug_object_activate(timer, &hrtimer_debug_descr);
}

static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
{
        debug_object_deactivate(timer, &hrtimer_debug_descr);
}

static inline void debug_hrtimer_assert_init(struct hrtimer *timer)
{
        debug_object_assert_init(timer, &hrtimer_debug_descr);
}

void destroy_hrtimer_on_stack(struct hrtimer *timer)
{
        debug_object_free(timer, &hrtimer_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);

#else

static inline void debug_hrtimer_init(struct hrtimer *timer) { }
static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { }
static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode) { }
static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
static inline void debug_hrtimer_assert_init(struct hrtimer *timer) { }
#endif

static inline void debug_setup(struct hrtimer *timer, clockid_t clockid, enum hrtimer_mode mode)
{
        debug_hrtimer_init(timer);
        trace_hrtimer_setup(timer, clockid, mode);
}

static inline void debug_setup_on_stack(struct hrtimer *timer, clockid_t clockid,
                                        enum hrtimer_mode mode)
{
        debug_hrtimer_init_on_stack(timer);
        trace_hrtimer_setup(timer, clockid, mode);
}

static inline void debug_activate(struct hrtimer *timer, enum hrtimer_mode mode, bool was_armed)
{
        debug_hrtimer_activate(timer, mode);
        trace_hrtimer_start(timer, mode, was_armed);
}

#define for_each_active_base(base, cpu_base, active)                                        \
        for (unsigned int idx = ffs(active); idx--; idx = ffs((active)))                \
                for (bool done = false; !done; active &= ~(1U << idx))                        \
                        for (base = &cpu_base->clock_base[idx]; !done; done = true)

#define hrtimer_from_timerqueue_node(_n) container_of_const(_n, struct hrtimer, node)

#if defined(CONFIG_NO_HZ_COMMON)
/*
 * Same as hrtimer_bases_next_event() below, but skips the excluded timer and
 * does not update cpu_base->next_timer/expires.
 */
static ktime_t hrtimer_bases_next_event_without(struct hrtimer_cpu_base *cpu_base,
                                                const struct hrtimer *exclude,
                                                unsigned int active, ktime_t expires_next)
{
        struct hrtimer_clock_base *base;
        ktime_t expires;

        lockdep_assert_held(&cpu_base->lock);

        for_each_active_base(base, cpu_base, active) {
                expires = ktime_sub(base->expires_next, base->offset);
                if (expires >= expires_next)
                        continue;

                /*
                 * If the excluded timer is the first on this base evaluate the
                 * next timer.
                 */
                struct timerqueue_linked_node *node = timerqueue_linked_first(&base->active);

                if (unlikely(&exclude->node == node)) {
                        node = timerqueue_linked_next(node);
                        if (!node)
                                continue;
                        expires = ktime_sub(node->expires, base->offset);
                        if (expires >= expires_next)
                                continue;
                }
                expires_next = expires;
        }
        /* If base->offset changed, the result might be negative */
        return max(expires_next, 0);
}
#endif

static __always_inline struct hrtimer *clock_base_next_timer(struct hrtimer_clock_base *base)
{
        struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);

        return hrtimer_from_timerqueue_node(next);
}

/* Find the base with the earliest expiry */
static void hrtimer_bases_first(struct hrtimer_cpu_base *cpu_base,unsigned int active,
                                ktime_t *expires_next, struct hrtimer **next_timer)
{
        struct hrtimer_clock_base *base;
        ktime_t expires;

        for_each_active_base(base, cpu_base, active) {
                expires = ktime_sub(base->expires_next, base->offset);
                if (expires < *expires_next) {
                        *expires_next = expires;
                        *next_timer = clock_base_next_timer(base);
                }
        }
}

/*
 * Recomputes cpu_base::*next_timer and returns the earliest expires_next
 * but does not set cpu_base::*expires_next, that is done by
 * hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating
 * cpu_base::*expires_next right away, reprogramming logic would no longer
 * work.
 *
 * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
 * those timers will get run whenever the softirq gets handled, at the end of
 * hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
 *
 * Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.
 * The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual
 * softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.
 *
 * @active_mask must be one of:
 *  - HRTIMER_ACTIVE_ALL,
 *  - HRTIMER_ACTIVE_SOFT, or
 *  - HRTIMER_ACTIVE_HARD.
 */
static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
{
        struct hrtimer *next_timer = NULL;
        ktime_t expires_next = KTIME_MAX;
        unsigned int active;

        lockdep_assert_held(&cpu_base->lock);

        if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
                active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
                if (active)
                        hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
                cpu_base->softirq_next_timer = next_timer;
        }

        if (active_mask & HRTIMER_ACTIVE_HARD) {
                active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
                if (active)
                        hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
                cpu_base->next_timer = next_timer;
        }
        return max(expires_next, 0);
}

static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
{
        ktime_t expires_next, soft = KTIME_MAX;

        /*
         * If the soft interrupt has already been activated, ignore the
         * soft bases. They will be handled in the already raised soft
         * interrupt.
         */
        if (!cpu_base->softirq_activated) {
                soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
                /*
                 * Update the soft expiry time. clock_settime() might have
                 * affected it.
                 */
                cpu_base->softirq_expires_next = soft;
        }

        expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);
        /*
         * If a softirq timer is expiring first, update cpu_base->next_timer
         * and program the hardware with the soft expiry time.
         */
        if (expires_next > soft) {
                cpu_base->next_timer = cpu_base->softirq_next_timer;
                expires_next = soft;
        }

        return expires_next;
}

static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
{
        ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
        ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
        ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;

        ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, offs_real,
                                                   offs_boot, offs_tai);

        base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
        base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
        base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;

        return now;
}

/*
 * Is the high resolution mode active in the CPU base. This cannot use the
 * static key as the CPUs are switched to high resolution mode
 * asynchronously.
 */
static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
{
        return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
                cpu_base->hres_active : 0;
}

static inline void hrtimer_rearm_event(ktime_t expires_next, bool deferred)
{
        trace_hrtimer_rearm(expires_next, deferred);
        tick_program_event(expires_next, 1);
}

static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer,
                                ktime_t expires_next)
{
        cpu_base->expires_next = expires_next;

        /*
         * If hres is not active, hardware does not have to be
         * reprogrammed yet.
         *
         * If a hang was detected in the last timer interrupt then we
         * leave the hang delay active in the hardware. We want the
         * system to make progress. That also prevents the following
         * scenario:
         * T1 expires 50ms from now
         * T2 expires 5s from now
         *
         * T1 is removed, so this code is called and would reprogram
         * the hardware to 5s from now. Any hrtimer_start after that
         * will not reprogram the hardware due to hang_detected being
         * set. So we'd effectively block all timers until the T2 event
         * fires.
         */
        if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
                return;

        hrtimer_rearm_event(expires_next, false);
}

/* Reprogram the event source with a evaluation of all clock bases */
static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip_equal)
{
        ktime_t expires_next = hrtimer_update_next_event(cpu_base);

        if (skip_equal && expires_next == cpu_base->expires_next)
                return;

        __hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);
}

/* High resolution timer related functions */
#ifdef CONFIG_HIGH_RES_TIMERS

/* High resolution timer enabled ? */
static bool hrtimer_hres_enabled __read_mostly  = true;
unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
EXPORT_SYMBOL_GPL(hrtimer_resolution);

/* Enable / Disable high resolution mode */
static int __init setup_hrtimer_hres(char *str)
{
        return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
}
__setup("highres=", setup_hrtimer_hres);

/* hrtimer_high_res_enabled - query, if the highres mode is enabled */
static inline bool hrtimer_is_hres_enabled(void)
{
        return hrtimer_hres_enabled;
}

/* Switch to high resolution mode */
static void hrtimer_switch_to_hres(void)
{
        struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);

        if (tick_init_highres()) {
                pr_warn("Could not switch to high resolution mode on CPU %u\n",        base->cpu);
                return;
        }
        base->hres_active = true;
        hrtimer_resolution = HIGH_RES_NSEC;

        tick_setup_sched_timer(true);
        /* "Retrigger" the interrupt to get things going */
        retrigger_next_event(NULL);
        hrtimer_schedule_hres_work();
}

#else

static inline bool hrtimer_is_hres_enabled(void) { return 0; }
static inline void hrtimer_switch_to_hres(void) { }

#endif /* CONFIG_HIGH_RES_TIMERS */

/*
 * Retrigger next event is called after clock was set with interrupts
 * disabled through an SMP function call or directly from low level
 * resume code.
 *
 * This is only invoked when:
 *        - CONFIG_HIGH_RES_TIMERS is enabled.
 *        - CONFIG_NOHZ_COMMON is enabled
 *
 * For the other cases this function is empty and because the call sites
 * are optimized out it vanishes as well, i.e. no need for lots of
 * #ifdeffery.
 */
static void retrigger_next_event(void *arg)
{
        struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);

        /*
         * When high resolution mode or nohz is active, then the offsets of
         * CLOCK_REALTIME/TAI/BOOTTIME have to be updated. Otherwise the
         * next tick will take care of that.
         *
         * If high resolution mode is active then the next expiring timer
         * must be reevaluated and the clock event device reprogrammed if
         * necessary.
         *
         * In the NOHZ case the update of the offset and the reevaluation
         * of the next expiring timer is enough. The return from the SMP
         * function call will take care of the reprogramming in case the
         * CPU was in a NOHZ idle sleep.
         *
         * In periodic low resolution mode, the next softirq expiration
         * must also be updated.
         */
        guard(raw_spinlock)(&base->lock);
        hrtimer_update_base(base);
        if (hrtimer_hres_active(base))
                hrtimer_force_reprogram(base, /* skip_equal */ false);
        else
                hrtimer_update_next_event(base);
}

/*
 * When a timer is enqueued and expires earlier than the already enqueued
 * timers, we have to check, whether it expires earlier than the timer for
 * which the clock event device was armed.
 *
 * Called with interrupts disabled and base->cpu_base.lock held
 */
static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
{
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
        struct hrtimer_clock_base *base = timer->base;
        ktime_t expires = hrtimer_get_expires(timer);

        WARN_ON_ONCE(expires < 0);

        expires = ktime_sub(expires, base->offset);
        /*
         * CLOCK_REALTIME timer might be requested with an absolute
         * expiry time which is less than base->offset. Set it to 0.
         */
        if (expires < 0)
                expires = 0;

        if (timer->is_soft) {
                /*
                 * soft hrtimer could be started on a remote CPU. In this
                 * case softirq_expires_next needs to be updated on the
                 * remote CPU. The soft hrtimer will not expire before the
                 * first hard hrtimer on the remote CPU -
                 * hrtimer_check_target() prevents this case.
                 */
                struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;

                if (timer_cpu_base->softirq_activated)
                        return;

                if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))
                        return;

                timer_cpu_base->softirq_next_timer = timer;
                timer_cpu_base->softirq_expires_next = expires;

                if (!ktime_before(expires, timer_cpu_base->expires_next) || !reprogram)
                        return;
        }

        /*
         * If the timer is not on the current cpu, we cannot reprogram
         * the other cpus clock event device.
         */
        if (base->cpu_base != cpu_base)
                return;

        if (expires >= cpu_base->expires_next)
                return;

        /* If a deferred rearm is pending skip reprogramming the device */
        if (cpu_base->deferred_rearm)
                return;

        cpu_base->next_timer = timer;

        __hrtimer_reprogram(cpu_base, timer, expires);
}

static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active)
{
        struct hrtimer_clock_base *base;
        unsigned int seq;
        ktime_t expires;

        /*
         * Update the base offsets unconditionally so the following
         * checks whether the SMP function call is required works.
         *
         * The update is safe even when the remote CPU is in the hrtimer
         * interrupt or the hrtimer soft interrupt and expiring affected
         * bases. Either it will see the update before handling a base or
         * it will see it when it finishes the processing and reevaluates
         * the next expiring timer.
         */
        seq = cpu_base->clock_was_set_seq;
        hrtimer_update_base(cpu_base);

        /*
         * If the sequence did not change over the update then the
         * remote CPU already handled it.
         */
        if (seq == cpu_base->clock_was_set_seq)
                return false;

        /* If a deferred rearm is pending the remote CPU will take care of it */
        if (cpu_base->deferred_rearm) {
                cpu_base->deferred_needs_update = true;
                return false;
        }

        /*
         * Walk the affected clock bases and check whether the first expiring
         * timer in a clock base is moving ahead of the first expiring timer of
         * @cpu_base. If so, the IPI must be invoked because per CPU clock
         * event devices cannot be remotely reprogrammed.
         */
        active &= cpu_base->active_bases;

        for_each_active_base(base, cpu_base, active) {
                struct timerqueue_linked_node *next;

                next = timerqueue_linked_first(&base->active);
                expires = ktime_sub(next->expires, base->offset);
                if (expires < cpu_base->expires_next)
                        return true;

                /* Extra check for softirq clock bases */
                if (base->index < HRTIMER_BASE_MONOTONIC_SOFT)
                        continue;
                if (cpu_base->softirq_activated)
                        continue;
                if (expires < cpu_base->softirq_expires_next)
                        return true;
        }
        return false;
}

/*
 * Clock was set. This might affect CLOCK_REALTIME, CLOCK_TAI and
 * CLOCK_BOOTTIME (for late sleep time injection).
 *
 * This requires to update the offsets for these clocks
 * vs. CLOCK_MONOTONIC. When high resolution timers are enabled, then this
 * also requires to eventually reprogram the per CPU clock event devices
 * when the change moves an affected timer ahead of the first expiring
 * timer on that CPU. Obviously remote per CPU clock event devices cannot
 * be reprogrammed. The other reason why an IPI has to be sent is when the
 * system is in !HIGH_RES and NOHZ mode. The NOHZ mode updates the offsets
 * in the tick, which obviously might be stopped, so this has to bring out
 * the remote CPU which might sleep in idle to get this sorted.
 */
void clock_was_set(unsigned int bases)
{
        cpumask_var_t mask;

        if (!hrtimer_highres_enabled() && !tick_nohz_is_active())
                goto out_timerfd;

        if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
                on_each_cpu(retrigger_next_event, NULL, 1);
                goto out_timerfd;
        }

        /* Avoid interrupting CPUs if possible */
        scoped_guard(cpus_read_lock) {
                int cpu;

                for_each_online_cpu(cpu) {
                        struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);

                        guard(raw_spinlock_irqsave)(&cpu_base->lock);
                        if (update_needs_ipi(cpu_base, bases))
                                cpumask_set_cpu(cpu, mask);
                }
                scoped_guard(preempt)
                        smp_call_function_many(mask, retrigger_next_event, NULL, 1);
        }
        free_cpumask_var(mask);

out_timerfd:
        timerfd_clock_was_set();
}

static void clock_was_set_work(struct work_struct *work)
{
        clock_was_set(CLOCK_SET_WALL);
}

static DECLARE_WORK(hrtimer_work, clock_was_set_work);

/*
 * Called from timekeeping code to reprogram the hrtimer interrupt device
 * on all cpus and to notify timerfd.
 */
void clock_was_set_delayed(void)
{
        schedule_work(&hrtimer_work);
}

/*
 * Called during resume either directly from via timekeeping_resume()
 * or in the case of s2idle from tick_unfreeze() to ensure that the
 * hrtimers are up to date.
 */
void hrtimers_resume_local(void)
{
        lockdep_assert_irqs_disabled();
        /* Retrigger on the local CPU */
        retrigger_next_event(NULL);
}

/* Counterpart to lock_hrtimer_base above */
static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
        __releases(&timer->base->cpu_base->lock)
{
        raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
}

/**
 * hrtimer_forward() - forward the timer expiry
 * @timer:        hrtimer to forward
 * @now:        forward past this time
 * @interval:        the interval to forward
 *
 * Forward the timer expiry so it will expire in the future.
 *
 * .. note::
 *  This only updates the timer expiry value and does not requeue the timer.
 *
 * There is also a variant of this function: hrtimer_forward_now().
 *
 * Context: Can be safely called from the callback function of @timer. If called
 *          from other contexts @timer must neither be enqueued nor running the
 *          callback and the caller needs to take care of serialization.
 *
 * Return: The number of overruns are returned.
 */
u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
{
        ktime_t delta;
        u64 orun = 1;

        delta = ktime_sub(now, hrtimer_get_expires(timer));

        if (delta < 0)
                return 0;

        if (WARN_ON(timer->is_queued))
                return 0;

        if (interval < hrtimer_resolution)
                interval = hrtimer_resolution;

        if (unlikely(delta >= interval)) {
                s64 incr = ktime_to_ns(interval);

                orun = ktime_divns(delta, incr);
                hrtimer_add_expires_ns(timer, incr * orun);
                if (hrtimer_get_expires(timer) > now)
                        return orun;
                /*
                 * This (and the ktime_add() below) is the
                 * correction for exact:
                 */
                orun++;
        }
        hrtimer_add_expires(timer, interval);

        return orun;
}
EXPORT_SYMBOL_GPL(hrtimer_forward);

/*
 * enqueue_hrtimer - internal function to (re)start a timer
 *
 * The timer is inserted in expiry order. Insertion into the
 * red black tree is O(log(n)).
 *
 * Returns true when the new timer is the leftmost timer in the tree.
 */
static bool enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
                            enum hrtimer_mode mode, bool was_armed)
{
        lockdep_assert_held(&base->cpu_base->lock);

        debug_activate(timer, mode, was_armed);
        WARN_ON_ONCE(!base->cpu_base->online);

        base->cpu_base->active_bases |= 1 << base->index;

        /* Pairs with the lockless read in hrtimer_is_queued() */
        WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);

        if (!timerqueue_linked_add(&base->active, &timer->node))
                return false;

        base->expires_next = hrtimer_get_expires(timer);
        return true;
}

static inline void base_update_next_timer(struct hrtimer_clock_base *base)
{
        struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);

        base->expires_next = next ? next->expires : KTIME_MAX;
}

/*
 * __remove_hrtimer - internal function to remove a timer
 *
 * High resolution timer mode reprograms the clock event device when the
 * timer is the one which expires next. The caller can disable this by setting
 * reprogram to zero. This is useful, when the context does a reprogramming
 * anyway (e.g. timer interrupt)
 */
static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
                             bool newstate, bool reprogram)
{
        struct hrtimer_cpu_base *cpu_base = base->cpu_base;
        bool was_first;

        lockdep_assert_held(&cpu_base->lock);

        if (!timer->is_queued)
                return;

        /* Pairs with the lockless read in hrtimer_is_queued() */
        WRITE_ONCE(timer->is_queued, newstate);

        was_first = !timerqueue_linked_prev(&timer->node);

        if (!timerqueue_linked_del(&base->active, &timer->node))
                cpu_base->active_bases &= ~(1 << base->index);

        /* Nothing to update if this was not the first timer in the base */
        if (!was_first)
                return;

        base_update_next_timer(base);

        /*
         * If reprogram is false don't update cpu_base->next_timer and do not
         * touch the clock event device.
         *
         * This happens when removing the first timer on a remote CPU, which
         * will be handled by the remote CPU's interrupt. It also happens when
         * a local timer is removed to be immediately restarted. That's handled
         * at the call site.
         */
        if (!reprogram || timer != cpu_base->next_timer || timer->is_lazy)
                return;

        if (cpu_base->deferred_rearm)
                cpu_base->deferred_needs_update = true;
        else
                hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
}

static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
                                  bool newstate)
{
        lockdep_assert_held(&base->cpu_base->lock);

        if (timer->is_queued) {
                bool reprogram;

                debug_hrtimer_deactivate(timer);

                /*
                 * Remove the timer and force reprogramming when high
                 * resolution mode is active and the timer is on the current
                 * CPU. If we remove a timer on another CPU, reprogramming is
                 * skipped. The interrupt event on this CPU is fired and
                 * reprogramming happens in the interrupt handler. This is a
                 * rare case and less expensive than a smp call.
                 */
                reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);

                __remove_hrtimer(timer, base, newstate, reprogram);
                return true;
        }
        return false;
}

/*
 * Update in place has to retrieve the expiry times of the neighbour nodes
 * if they exist. That is cache line neutral because the dequeue/enqueue
 * operation is going to need the same cache lines. But there is a big win
 * when the dequeue/enqueue can be avoided because the RB tree does not
 * have to be rebalanced twice.
 */
static inline bool
hrtimer_can_update_in_place(struct hrtimer *timer, struct hrtimer_clock_base *base, ktime_t expires)
{
        struct timerqueue_linked_node *next = timerqueue_linked_next(&timer->node);
        struct timerqueue_linked_node *prev = timerqueue_linked_prev(&timer->node);

        /* If the new expiry goes behind the next timer, requeue is required */
        if (next && expires > next->expires)
                return false;

        /* If this is the first timer, update in place */
        if (!prev)
                return true;

        /* Update in place when it does not go ahead of the previous one */
        return expires >= prev->expires;
}

static inline bool
remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
                             const enum hrtimer_mode mode, ktime_t expires, u64 delta_ns)
{
        bool was_first = false;

        /* Remove it from the timer queue if active */
        if (timer->is_queued) {
                was_first = !timerqueue_linked_prev(&timer->node);

                /* Try to update in place to avoid the de/enqueue dance */
                if (hrtimer_can_update_in_place(timer, base, expires)) {
                        hrtimer_set_expires_range_ns(timer, expires, delta_ns);
                        trace_hrtimer_start(timer, mode, true);
                        if (was_first)
                                base->expires_next = expires;
                        return was_first;
                }

                debug_hrtimer_deactivate(timer);
                timerqueue_linked_del(&base->active, &timer->node);
        }

        /* Set the new expiry time */
        hrtimer_set_expires_range_ns(timer, expires, delta_ns);

        debug_activate(timer, mode, timer->is_queued);
        base->cpu_base->active_bases |= 1 << base->index;

        /* Pairs with the lockless read in hrtimer_is_queued() */
        WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);

        /* If it's the first expiring timer now or again, update base */
        if (timerqueue_linked_add(&base->active, &timer->node)) {
                base->expires_next = expires;
                return true;
        }

        if (was_first)
                base_update_next_timer(base);

        return false;
}

static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
                                            const enum hrtimer_mode mode)
{
#ifdef CONFIG_TIME_LOW_RES
        /*
         * CONFIG_TIME_LOW_RES indicates that the system has no way to return
         * granular time values. For relative timers we add hrtimer_resolution
         * (i.e. one jiffy) to prevent short timeouts.
         */
        timer->is_rel = mode & HRTIMER_MODE_REL;
        if (timer->is_rel)
                tim = ktime_add_safe(tim, hrtimer_resolution);
#endif
        return tim;
}

static void hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
{
        ktime_t expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);

        /*
         * Reprogramming needs to be triggered, even if the next soft
         * hrtimer expires at the same time as the next hard
         * hrtimer. cpu_base->softirq_expires_next needs to be updated!
         */
        if (expires == KTIME_MAX)
                return;

        /*
         * cpu_base->next_timer is recomputed by __hrtimer_get_next_event()
         * cpu_base->expires_next is only set by hrtimer_reprogram()
         */
        hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
}

#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
static __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned)
{
        if (static_branch_likely(&timers_migration_enabled)) {
                /*
                 * If it is local and the first expiring timer keep it on the local
                 * CPU to optimize reprogramming of the clockevent device. Also
                 * avoid switch_hrtimer_base() overhead when local and pinned.
                 */
                if (!is_local)
                        return false;
                if (is_first || is_pinned)
                        return true;

                /* Honour the NOHZ full restrictions */
                if (!housekeeping_cpu(smp_processor_id(), HK_TYPE_KERNEL_NOISE))
                        return false;

                /*
                 * If the tick is not stopped or need_resched() is set, then
                 * there is no point in moving the timer somewhere else.
                 */
                return !tick_nohz_tick_stopped() || need_resched();
        }
        return is_local;
}
#else
static __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned)
{
        return is_local;
}
#endif

static inline bool hrtimer_keep_base(struct hrtimer *timer, bool is_local, bool is_first,
                                     bool is_pinned)
{
        /* If the timer is running the callback it has to stay on its CPU base. */
        if (unlikely(timer->base->running == timer))
                return true;

        return hrtimer_prefer_local(is_local, is_first, is_pinned);
}

static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
                                     const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
{
        struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases);
        bool is_pinned, first, was_first, keep_base = false;
        struct hrtimer_cpu_base *cpu_base = base->cpu_base;

        was_first = cpu_base->next_timer == timer;
        is_pinned = !!(mode & HRTIMER_MODE_PINNED);

        /*
         * Don't keep it local if this enqueue happens on a unplugged CPU
         * after hrtimer_cpu_dying() has been invoked.
         */
        if (likely(this_cpu_base->online)) {
                bool is_local = cpu_base == this_cpu_base;

                keep_base = hrtimer_keep_base(timer, is_local, was_first, is_pinned);
        }

        /* Calculate absolute expiry time for relative timers */
        if (mode & HRTIMER_MODE_REL)
                tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
        /* Compensate for low resolution granularity */
        tim = hrtimer_update_lowres(timer, tim, mode);

        /*
         * Remove an active timer from the queue. In case it is not queued
         * on the current CPU, make sure that remove_hrtimer() updates the
         * remote data correctly.
         *
         * If it's on the current CPU and the first expiring timer, then
         * skip reprogramming, keep the timer local and enforce
         * reprogramming later if it was the first expiring timer.  This
         * avoids programming the underlying clock event twice (once at
         * removal and once after enqueue).
         *
         * @keep_base is also true if the timer callback is running on a
         * remote CPU and for local pinned timers.
         */
        if (likely(keep_base)) {
                first = remove_and_enqueue_same_base(timer, base, mode, tim, delta_ns);
        } else {
                /* Keep the ENQUEUED state in case it is queued */
                bool was_armed = remove_hrtimer(timer, base, HRTIMER_STATE_ENQUEUED);

                hrtimer_set_expires_range_ns(timer, tim, delta_ns);

                /* Switch the timer base, if necessary: */
                base = switch_hrtimer_base(timer, base, is_pinned);
                cpu_base = base->cpu_base;

                first = enqueue_hrtimer(timer, base, mode, was_armed);
        }

        /* If a deferred rearm is pending skip reprogramming the device */
        if (cpu_base->deferred_rearm) {
                cpu_base->deferred_needs_update = true;
                return false;
        }

        if (!was_first || cpu_base != this_cpu_base) {
                /*
                 * If the current CPU base is online, then the timer is never
                 * queued on a remote CPU if it would be the first expiring
                 * timer there unless the timer callback is currently executed
                 * on the remote CPU. In the latter case the remote CPU will
                 * re-evaluate the first expiring timer after completing the
                 * callbacks.
                 */
                if (likely(hrtimer_base_is_online(this_cpu_base)))
                        return first;

                /*
                 * Timer was enqueued remote because the current base is
                 * already offline. If the timer is the first to expire,
                 * kick the remote CPU to reprogram the clock event.
                 */
                if (first)
                        smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd);
                return false;
        }

        /*
         * Special case for the HRTICK timer. It is frequently rearmed and most
         * of the time moves the expiry into the future. That's expensive in
         * virtual machines and it's better to take the pointless already armed
         * interrupt than reprogramming the hardware on every context switch.
         *
         * If the new expiry is before the armed time, then reprogramming is
         * required.
         */
        if (timer->is_lazy) {
                if (cpu_base->expires_next <= hrtimer_get_expires(timer))
                        return false;
        }

        /*
         * Timer was the first expiring timer and forced to stay on the
         * current CPU to avoid reprogramming on removal and enqueue. Force
         * reprogram the hardware by evaluating the new first expiring
         * timer.
         */
        hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
        return false;
}

/**
 * hrtimer_start_range_ns - (re)start an hrtimer
 * @timer:        the timer to be added
 * @tim:        expiry time
 * @delta_ns:        "slack" range for the timer
 * @mode:        timer mode: absolute (HRTIMER_MODE_ABS) or
 *                relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
 *                softirq based mode is considered for debug purpose only!
 */
void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
                            const enum hrtimer_mode mode)
{
        struct hrtimer_clock_base *base;
        unsigned long flags;

        debug_hrtimer_assert_init(timer);

        /*
         * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
         * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
         * expiry mode because unmarked timers are moved to softirq expiry.
         */
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
        else
                WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);

        base = lock_hrtimer_base(timer, &flags);

        if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
                hrtimer_reprogram(timer, true);

        unlock_hrtimer_base(timer, &flags);
}
EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);

/**
 * hrtimer_try_to_cancel - try to deactivate a timer
 * @timer:        hrtimer to stop
 *
 * Returns:
 *
 *  *  0 when the timer was not active
 *  *  1 when the timer was active
 *  * -1 when the timer is currently executing the callback function and
 *    cannot be stopped
 */
int hrtimer_try_to_cancel(struct hrtimer *timer)
{
        struct hrtimer_clock_base *base;
        unsigned long flags;
        int ret = -1;

        /*
         * Check lockless first. If the timer is not active (neither
         * enqueued nor running the callback, nothing to do here.  The
         * base lock does not serialize against a concurrent enqueue,
         * so we can avoid taking it.
         */
        if (!hrtimer_active(timer))
                return 0;

        base = lock_hrtimer_base(timer, &flags);

        if (!hrtimer_callback_running(timer)) {
                ret = remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE);
                if (ret)
                        trace_hrtimer_cancel(timer);
        }

        unlock_hrtimer_base(timer, &flags);

        return ret;

}
EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);

#ifdef CONFIG_PREEMPT_RT
static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base)
{
        spin_lock_init(&base->softirq_expiry_lock);
}

static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base)
        __acquires(&base->softirq_expiry_lock)
{
        spin_lock(&base->softirq_expiry_lock);
}

static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
        __releases(&base->softirq_expiry_lock)
{
        spin_unlock(&base->softirq_expiry_lock);
}

/*
 * The counterpart to hrtimer_cancel_wait_running().
 *
 * If there is a waiter for cpu_base->expiry_lock, then it was waiting for
 * the timer callback to finish. Drop expiry_lock and reacquire it. That
 * allows the waiter to acquire the lock and make progress.
 */
static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base, unsigned long flags)
{
        if (atomic_read(&cpu_base->timer_waiters)) {
                raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
                spin_unlock(&cpu_base->softirq_expiry_lock);
                spin_lock(&cpu_base->softirq_expiry_lock);
                raw_spin_lock_irq(&cpu_base->lock);
        }
}

#ifdef CONFIG_SMP
static __always_inline bool is_migration_base(struct hrtimer_clock_base *base)
{
        return base == &migration_base;
}
#else
static __always_inline bool is_migration_base(struct hrtimer_clock_base *base)
{
        return false;
}
#endif

/*
 * This function is called on PREEMPT_RT kernels when the fast path
 * deletion of a timer failed because the timer callback function was
 * running.
 *
 * This prevents priority inversion: if the soft irq thread is preempted
 * in the middle of a timer callback, then calling hrtimer_cancel() can
 * lead to two issues:
 *
 *  - If the caller is on a remote CPU then it has to spin wait for the timer
 *    handler to complete. This can result in unbound priority inversion.
 *
 *  - If the caller originates from the task which preempted the timer
 *    handler on the same CPU, then spin waiting for the timer handler to
 *    complete is never going to end.
 */
void hrtimer_cancel_wait_running(const struct hrtimer *timer)
{
        /* Lockless read. Prevent the compiler from reloading it below */
        struct hrtimer_clock_base *base = READ_ONCE(timer->base);

        /*
         * Just relax if the timer expires in hard interrupt context or if
         * it is currently on the migration base.
         */
        if (!timer->is_soft || is_migration_base(base)) {
                cpu_relax();
                return;
        }

        /*
         * Mark the base as contended and grab the expiry lock, which is
         * held by the softirq across the timer callback. Drop the lock
         * immediately so the softirq can expire the next timer. In theory
         * the timer could already be running again, but that's more than
         * unlikely and just causes another wait loop.
         */
        atomic_inc(&base->cpu_base->timer_waiters);
        spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
        atomic_dec(&base->cpu_base->timer_waiters);
        spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
}
#else
static inline void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
static inline void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
static inline void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base, unsigned long fl) { }
#endif

/**
 * hrtimer_cancel - cancel a timer and wait for the handler to finish.
 * @timer:        the timer to be cancelled
 *
 * Returns:
 *  0 when the timer was not active
 *  1 when the timer was active
 */
int hrtimer_cancel(struct hrtimer *timer)
{
        int ret;

        do {
                ret = hrtimer_try_to_cancel(timer);

                if (ret < 0)
                        hrtimer_cancel_wait_running(timer);
        } while (ret < 0);
        return ret;
}
EXPORT_SYMBOL_GPL(hrtimer_cancel);

/**
 * __hrtimer_get_remaining - get remaining time for the timer
 * @timer:        the timer to read
 * @adjust:        adjust relative timers when CONFIG_TIME_LOW_RES=y
 */
ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
{
        unsigned long flags;
        ktime_t rem;

        lock_hrtimer_base(timer, &flags);
        if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust)
                rem = hrtimer_expires_remaining_adjusted(timer);
        else
                rem = hrtimer_expires_remaining(timer);
        unlock_hrtimer_base(timer, &flags);

        return rem;
}
EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);

#ifdef CONFIG_NO_HZ_COMMON
/**
 * hrtimer_get_next_event - get the time until next expiry event
 *
 * Returns the next expiry time or KTIME_MAX if no timer is pending.
 */
u64 hrtimer_get_next_event(void)
{
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
        u64 expires = KTIME_MAX;

        guard(raw_spinlock_irqsave)(&cpu_base->lock);
        if (!hrtimer_hres_active(cpu_base))
                expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);

        return expires;
}

/**
 * hrtimer_next_event_without - time until next expiry event w/o one timer
 * @exclude:        timer to exclude
 *
 * Returns the next expiry time over all timers except for the @exclude one or
 * KTIME_MAX if none of them is pending.
 */
u64 hrtimer_next_event_without(const struct hrtimer *exclude)
{
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
        u64 expires = KTIME_MAX;
        unsigned int active;

        guard(raw_spinlock_irqsave)(&cpu_base->lock);
        if (!hrtimer_hres_active(cpu_base))
                return expires;

        active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
        if (active && !cpu_base->softirq_activated)
                expires = hrtimer_bases_next_event_without(cpu_base, exclude, active, KTIME_MAX);

        active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
        if (!active)
                return expires;
        return hrtimer_bases_next_event_without(cpu_base, exclude, active, expires);
}
#endif

static inline int hrtimer_clockid_to_base(clockid_t clock_id)
{
        switch (clock_id) {
        case CLOCK_MONOTONIC:
                return HRTIMER_BASE_MONOTONIC;
        case CLOCK_REALTIME:
                return HRTIMER_BASE_REALTIME;
        case CLOCK_BOOTTIME:
                return HRTIMER_BASE_BOOTTIME;
        case CLOCK_TAI:
                return HRTIMER_BASE_TAI;
        default:
                WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
                return HRTIMER_BASE_MONOTONIC;
        }
}

static ktime_t __hrtimer_cb_get_time(clockid_t clock_id)
{
        switch (clock_id) {
        case CLOCK_MONOTONIC:
                return ktime_get();
        case CLOCK_REALTIME:
                return ktime_get_real();
        case CLOCK_BOOTTIME:
                return ktime_get_boottime();
        case CLOCK_TAI:
                return ktime_get_clocktai();
        default:
                WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
                return ktime_get();
        }
}

ktime_t hrtimer_cb_get_time(const struct hrtimer *timer)
{
        return __hrtimer_cb_get_time(timer->base->clockid);
}
EXPORT_SYMBOL_GPL(hrtimer_cb_get_time);

static void __hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*fn)(struct hrtimer *),
                            clockid_t clock_id, enum hrtimer_mode mode)
{
        bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
        struct hrtimer_cpu_base *cpu_base;
        int base;

        /*
         * On PREEMPT_RT enabled kernels hrtimers which are not explicitly
         * marked for hard interrupt expiry mode are moved into soft
         * interrupt context for latency reasons and because the callbacks
         * can invoke functions which might sleep on RT, e.g. spin_lock().
         */
        if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD))
                softtimer = true;

        memset(timer, 0, sizeof(struct hrtimer));

        cpu_base = raw_cpu_ptr(&hrtimer_bases);

        /*
         * POSIX magic: Relative CLOCK_REALTIME timers are not affected by
         * clock modifications, so they needs to become CLOCK_MONOTONIC to
         * ensure POSIX compliance.
         */
        if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
                clock_id = CLOCK_MONOTONIC;

        base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
        base += hrtimer_clockid_to_base(clock_id);
        timer->is_soft = softtimer;
        timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
        timer->is_lazy = !!(mode & HRTIMER_MODE_LAZY_REARM);
        timer->base = &cpu_base->clock_base[base];
        timerqueue_linked_init(&timer->node);

        if (WARN_ON_ONCE(!fn))
                ACCESS_PRIVATE(timer, function) = hrtimer_dummy_timeout;
        else
                ACCESS_PRIVATE(timer, function) = fn;
}

/**
 * hrtimer_setup - initialize a timer to the given clock
 * @timer:        the timer to be initialized
 * @function:        the callback function
 * @clock_id:        the clock to be used
 * @mode:       The modes which are relevant for initialization:
 *              HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
 *              HRTIMER_MODE_REL_SOFT
 *
 *              The PINNED variants of the above can be handed in,
 *              but the PINNED bit is ignored as pinning happens
 *              when the hrtimer is started
 */
void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *),
                   clockid_t clock_id, enum hrtimer_mode mode)
{
        debug_setup(timer, clock_id, mode);
        __hrtimer_setup(timer, function, clock_id, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_setup);

/**
 * hrtimer_setup_on_stack - initialize a timer on stack memory
 * @timer:        The timer to be initialized
 * @function:        the callback function
 * @clock_id:        The clock to be used
 * @mode:       The timer mode
 *
 * Similar to hrtimer_setup(), except that this one must be used if struct hrtimer is in stack
 * memory.
 */
void hrtimer_setup_on_stack(struct hrtimer *timer,
                            enum hrtimer_restart (*function)(struct hrtimer *),
                            clockid_t clock_id, enum hrtimer_mode mode)
{
        debug_setup_on_stack(timer, clock_id, mode);
        __hrtimer_setup(timer, function, clock_id, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_setup_on_stack);

/*
 * A timer is active, when it is enqueued into the rbtree or the
 * callback function is running or it's in the state of being migrated
 * to another cpu.
 *
 * It is important for this function to not return a false negative.
 */
bool hrtimer_active(const struct hrtimer *timer)
{
        struct hrtimer_clock_base *base;
        unsigned int seq;

        do {
                base = READ_ONCE(timer->base);
                seq = raw_read_seqcount_begin(&base->seq);

                if (timer->is_queued || base->running == timer)
                        return true;

        } while (read_seqcount_retry(&base->seq, seq) || base != READ_ONCE(timer->base));

        return false;
}
EXPORT_SYMBOL_GPL(hrtimer_active);

/*
 * The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3
 * distinct sections:
 *
 *  - queued:        the timer is queued
 *  - callback:        the timer is being ran
 *  - post:        the timer is inactive or (re)queued
 *
 * On the read side we ensure we observe timer->is_queued and cpu_base->running
 * from the same section, if anything changed while we looked at it, we retry.
 * This includes timer->base changing because sequence numbers alone are
 * insufficient for that.
 *
 * The sequence numbers are required because otherwise we could still observe
 * a false negative if the read side got smeared over multiple consecutive
 * __run_hrtimer() invocations.
 */
static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_clock_base *base,
                          struct hrtimer *timer, ktime_t now, unsigned long flags)
        __must_hold(&cpu_base->lock)
{
        enum hrtimer_restart (*fn)(struct hrtimer *);
        bool expires_in_hardirq;
        int restart;

        lockdep_assert_held(&cpu_base->lock);

        debug_hrtimer_deactivate(timer);
        base->running = timer;

        /*
         * Separate the ->running assignment from the ->is_queued assignment.
         *
         * As with a regular write barrier, this ensures the read side in
         * hrtimer_active() cannot observe base->running == NULL &&
         * timer->is_queued == INACTIVE.
         */
        raw_write_seqcount_barrier(&base->seq);

        __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, false);
        fn = ACCESS_PRIVATE(timer, function);

        /*
         * Clear the 'is relative' flag for the TIME_LOW_RES case. If the
         * timer is restarted with a period then it becomes an absolute
         * timer. If its not restarted it does not matter.
         */
        if (IS_ENABLED(CONFIG_TIME_LOW_RES))
                timer->is_rel = false;

        /*
         * The timer is marked as running in the CPU base, so it is
         * protected against migration to a different CPU even if the lock
         * is dropped.
         */
        raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
        trace_hrtimer_expire_entry(timer, now);
        expires_in_hardirq = lockdep_hrtimer_enter(timer);

        restart = fn(timer);

        lockdep_hrtimer_exit(expires_in_hardirq);
        trace_hrtimer_expire_exit(timer);
        raw_spin_lock_irq(&cpu_base->lock);

        /*
         * Note: We clear the running state after enqueue_hrtimer and
         * we do not reprogram the event hardware. Happens either in
         * hrtimer_start_range_ns() or in hrtimer_interrupt()
         *
         * Note: Because we dropped the cpu_base->lock above,
         * hrtimer_start_range_ns() can have popped in and enqueued the timer
         * for us already.
         */
        if (restart == HRTIMER_RESTART && !timer->is_queued)
                enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS, false);

        /*
         * Separate the ->running assignment from the ->is_queued assignment.
         *
         * As with a regular write barrier, this ensures the read side in
         * hrtimer_active() cannot observe base->running.timer == NULL &&
         * timer->is_queued == INACTIVE.
         */
        raw_write_seqcount_barrier(&base->seq);

        WARN_ON_ONCE(base->running != timer);
        base->running = NULL;
}

static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base)
{
        struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);

        return next ? hrtimer_from_timerqueue_node(next) : NULL;
}

static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
                                 unsigned long flags, unsigned int active_mask)
{
        unsigned int active = cpu_base->active_bases & active_mask;
        struct hrtimer_clock_base *base;

        for_each_active_base(base, cpu_base, active) {
                ktime_t basenow = ktime_add(now, base->offset);
                struct hrtimer *timer;

                while ((timer = clock_base_next_timer(base))) {
                        /*
                         * The immediate goal for using the softexpires is
                         * minimizing wakeups, not running timers at the
                         * earliest interrupt after their soft expiration.
                         * This allows us to avoid using a Priority Search
                         * Tree, which can answer a stabbing query for
                         * overlapping intervals and instead use the simple
                         * BST we already have.
                         * We don't add extra wakeups by delaying timers that
                         * are right-of a not yet expired timer, because that
                         * timer will have to trigger a wakeup anyway.
                         */
                        if (basenow < hrtimer_get_softexpires(timer))
                                break;

                        __run_hrtimer(cpu_base, base, timer, basenow, flags);
                        if (active_mask == HRTIMER_ACTIVE_SOFT)
                                hrtimer_sync_wait_running(cpu_base, flags);
                }
        }
}

static __latent_entropy void hrtimer_run_softirq(void)
{
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
        unsigned long flags;
        ktime_t now;

        hrtimer_cpu_base_lock_expiry(cpu_base);
        raw_spin_lock_irqsave(&cpu_base->lock, flags);

        now = hrtimer_update_base(cpu_base);
        __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);

        cpu_base->softirq_activated = false;
        hrtimer_update_softirq_timer(cpu_base, true);

        raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
        hrtimer_cpu_base_unlock_expiry(cpu_base);
}

#ifdef CONFIG_HIGH_RES_TIMERS

/*
 * Very similar to hrtimer_force_reprogram(), except it deals with
 * deferred_rearm and hang_detected.
 */
static void hrtimer_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next, bool deferred)
{
        cpu_base->expires_next = expires_next;
        cpu_base->deferred_rearm = false;

        if (unlikely(cpu_base->hang_detected)) {
                /*
                 * Give the system a chance to do something else than looping
                 * on hrtimer interrupts.
                 */
                expires_next = ktime_add_ns(ktime_get(),
                                            min(100 * NSEC_PER_MSEC, cpu_base->max_hang_time));
        }
        hrtimer_rearm_event(expires_next, deferred);
}

#ifdef CONFIG_HRTIMER_REARM_DEFERRED
void __hrtimer_rearm_deferred(void)
{
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
        ktime_t expires_next;

        if (!cpu_base->deferred_rearm)
                return;

        guard(raw_spinlock)(&cpu_base->lock);
        if (cpu_base->deferred_needs_update) {
                hrtimer_update_base(cpu_base);
                expires_next = hrtimer_update_next_event(cpu_base);
        } else {
                /* No timer added/removed. Use the cached value */
                expires_next = cpu_base->deferred_expires_next;
        }
        hrtimer_rearm(cpu_base, expires_next, true);
}

static __always_inline void
hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
{
        /* hrtimer_interrupt() just re-evaluated the first expiring timer */
        cpu_base->deferred_needs_update = false;
        /* Cache the expiry time */
        cpu_base->deferred_expires_next = expires_next;
        set_thread_flag(TIF_HRTIMER_REARM);
}
#else  /* CONFIG_HRTIMER_REARM_DEFERRED */
static __always_inline void
hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
{
        hrtimer_rearm(cpu_base, expires_next, false);
}
#endif  /* !CONFIG_HRTIMER_REARM_DEFERRED */

/*
 * High resolution timer interrupt
 * Called with interrupts disabled
 */
void hrtimer_interrupt(struct clock_event_device *dev)
{
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
        ktime_t expires_next, now, entry_time, delta;
        unsigned long flags;
        int retries = 0;

        BUG_ON(!cpu_base->hres_active);
        cpu_base->nr_events++;
        dev->next_event = KTIME_MAX;
        dev->next_event_forced = 0;

        raw_spin_lock_irqsave(&cpu_base->lock, flags);
        entry_time = now = hrtimer_update_base(cpu_base);
retry:
        cpu_base->deferred_rearm = true;
        /*
         * Set expires_next to KTIME_MAX, which prevents that remote CPUs queue
         * timers while __hrtimer_run_queues() is expiring the clock bases.
         * Timers which are re/enqueued on the local CPU are not affected by
         * this.
         */
        cpu_base->expires_next = KTIME_MAX;

        if (!ktime_before(now, cpu_base->softirq_expires_next)) {
                cpu_base->softirq_expires_next = KTIME_MAX;
                cpu_base->softirq_activated = true;
                raise_timer_softirq(HRTIMER_SOFTIRQ);
        }

        __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);

        /*
         * The next timer was already expired due to:
         * - tracing
         * - long lasting callbacks
         * - being scheduled away when running in a VM
         *
         * We need to prevent that we loop forever in the hrtiner interrupt
         * routine. We give it 3 attempts to avoid overreacting on some
         * spurious event.
         */
        now = hrtimer_update_base(cpu_base);
        expires_next = hrtimer_update_next_event(cpu_base);
        cpu_base->hang_detected = false;
        if (expires_next < now) {
                if (++retries < 3)
                        goto retry;

                delta = ktime_sub(now, entry_time);
                cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta);
                cpu_base->nr_hangs++;
                cpu_base->hang_detected = true;
        }

        hrtimer_interrupt_rearm(cpu_base, expires_next);
        raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
}

#endif /* !CONFIG_HIGH_RES_TIMERS */

/*
 * Called from run_local_timers in hardirq context every jiffy
 */
void hrtimer_run_queues(void)
{
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
        unsigned long flags;
        ktime_t now;

        if (hrtimer_hres_active(cpu_base))
                return;

        /*
         * This _is_ ugly: We have to check periodically, whether we
         * can switch to highres and / or nohz mode. The clocksource
         * switch happens with xtime_lock held. Notification from
         * there only sets the check bit in the tick_oneshot code,
         * otherwise we might deadlock vs. xtime_lock.
         */
        if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) {
                hrtimer_switch_to_hres();
                return;
        }

        raw_spin_lock_irqsave(&cpu_base->lock, flags);
        now = hrtimer_update_base(cpu_base);

        if (!ktime_before(now, cpu_base->softirq_expires_next)) {
                cpu_base->softirq_expires_next = KTIME_MAX;
                cpu_base->softirq_activated = true;
                raise_timer_softirq(HRTIMER_SOFTIRQ);
        }

        __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
        raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
}

/*
 * Sleep related functions:
 */
static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
{
        struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer);
        struct task_struct *task = t->task;

        t->task = NULL;
        if (task)
                wake_up_process(task);

        return HRTIMER_NORESTART;
}

/**
 * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
 * @sl:                sleeper to be started
 * @mode:        timer mode abs/rel
 *
 * Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
 * to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
 */
void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode mode)
{
        /*
         * Make the enqueue delivery mode check work on RT. If the sleeper
         * was initialized for hard interrupt delivery, force the mode bit.
         * This is a special case for hrtimer_sleepers because
         * __hrtimer_setup_sleeper() determines the delivery mode on RT so the
         * fiddling with this decision is avoided at the call sites.
         */
        if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
                mode |= HRTIMER_MODE_HARD;

        hrtimer_start_expires(&sl->timer, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);

static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
                                    enum hrtimer_mode mode)
{
        /*
         * On PREEMPT_RT enabled kernels hrtimers which are not explicitly
         * marked for hard interrupt expiry mode are moved into soft
         * interrupt context either for latency reasons or because the
         * hrtimer callback takes regular spinlocks or invokes other
         * functions which are not suitable for hard interrupt context on
         * PREEMPT_RT.
         *
         * The hrtimer_sleeper callback is RT compatible in hard interrupt
         * context, but there is a latency concern: Untrusted userspace can
         * spawn many threads which arm timers for the same expiry time on
         * the same CPU. That causes a latency spike due to the wakeup of
         * a gazillion threads.
         *
         * OTOH, privileged real-time user space applications rely on the
         * low latency of hard interrupt wakeups. If the current task is in
         * a real-time scheduling class, mark the mode for hard interrupt
         * expiry.
         */
        if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
                if (rt_or_dl_task_policy(current) && !(mode & HRTIMER_MODE_SOFT))
                        mode |= HRTIMER_MODE_HARD;
        }

        __hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode);
        sl->task = current;
}

/**
 * hrtimer_setup_sleeper_on_stack - initialize a sleeper in stack memory
 * @sl:                sleeper to be initialized
 * @clock_id:        the clock to be used
 * @mode:        timer mode abs/rel
 */
void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl, clockid_t clock_id,
                                    enum hrtimer_mode mode)
{
        debug_setup_on_stack(&sl->timer, clock_id, mode);
        __hrtimer_setup_sleeper(sl, clock_id, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_setup_sleeper_on_stack);

int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
{
        switch(restart->nanosleep.type) {
#ifdef CONFIG_COMPAT_32BIT_TIME
        case TT_COMPAT:
                if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp))
                        return -EFAULT;
                break;
#endif
        case TT_NATIVE:
                if (put_timespec64(ts, restart->nanosleep.rmtp))
                        return -EFAULT;
                break;
        default:
                BUG();
        }
        return -ERESTART_RESTARTBLOCK;
}

static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
{
        struct restart_block *restart;

        do {
                set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
                hrtimer_sleeper_start_expires(t, mode);

                if (likely(t->task))
                        schedule();

                hrtimer_cancel(&t->timer);
                mode = HRTIMER_MODE_ABS;

        } while (t->task && !signal_pending(current));

        __set_current_state(TASK_RUNNING);

        if (!t->task)
                return 0;

        restart = &current->restart_block;
        if (restart->nanosleep.type != TT_NONE) {
                ktime_t rem = hrtimer_expires_remaining(&t->timer);
                struct timespec64 rmt;

                if (rem <= 0)
                        return 0;
                rmt = ktime_to_timespec64(rem);

                return nanosleep_copyout(restart, &rmt);
        }
        return -ERESTART_RESTARTBLOCK;
}

static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
{
        struct hrtimer_sleeper t;
        int ret;

        hrtimer_setup_sleeper_on_stack(&t, restart->nanosleep.clockid, HRTIMER_MODE_ABS);
        hrtimer_set_expires(&t.timer, restart->nanosleep.expires);
        ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
        destroy_hrtimer_on_stack(&t.timer);
        return ret;
}

long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, const clockid_t clockid)
{
        struct restart_block *restart;
        struct hrtimer_sleeper t;
        int ret;

        hrtimer_setup_sleeper_on_stack(&t, clockid, mode);
        hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns);
        ret = do_nanosleep(&t, mode);
        if (ret != -ERESTART_RESTARTBLOCK)
                goto out;

        /* Absolute timers do not update the rmtp value and restart: */
        if (mode == HRTIMER_MODE_ABS) {
                ret = -ERESTARTNOHAND;
                goto out;
        }

        restart = &current->restart_block;
        restart->nanosleep.clockid = t.timer.base->clockid;
        restart->nanosleep.expires = hrtimer_get_expires(&t.timer);
        set_restart_fn(restart, hrtimer_nanosleep_restart);
out:
        destroy_hrtimer_on_stack(&t.timer);
        return ret;
}

#ifdef CONFIG_64BIT

SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
                struct __kernel_timespec __user *, rmtp)
{
        struct timespec64 tu;

        if (get_timespec64(&tu, rqtp))
                return -EFAULT;

        if (!timespec64_valid(&tu))
                return -EINVAL;

        current->restart_block.fn = do_no_restart_syscall;
        current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
        current->restart_block.nanosleep.rmtp = rmtp;
        return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, CLOCK_MONOTONIC);
}

#endif

#ifdef CONFIG_COMPAT_32BIT_TIME

SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
                struct old_timespec32 __user *, rmtp)
{
        struct timespec64 tu;

        if (get_old_timespec32(&tu, rqtp))
                return -EFAULT;

        if (!timespec64_valid(&tu))
                return -EINVAL;

        current->restart_block.fn = do_no_restart_syscall;
        current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
        current->restart_block.nanosleep.compat_rmtp = rmtp;
        return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, CLOCK_MONOTONIC);
}
#endif

/*
 * Functions related to boot-time initialization:
 */
int hrtimers_prepare_cpu(unsigned int cpu)
{
        struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);

        for (int i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
                struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];

                clock_b->cpu_base = cpu_base;
                seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);
                timerqueue_linked_init_head(&clock_b->active);
        }

        cpu_base->cpu = cpu;
        hrtimer_cpu_base_init_expiry_lock(cpu_base);
        return 0;
}

int hrtimers_cpu_starting(unsigned int cpu)
{
        struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);

        /* Clear out any left over state from a CPU down operation */
        cpu_base->active_bases = 0;
        cpu_base->hres_active = false;
        cpu_base->hang_detected = false;
        cpu_base->next_timer = NULL;
        cpu_base->softirq_next_timer = NULL;
        cpu_base->expires_next = KTIME_MAX;
        cpu_base->softirq_expires_next = KTIME_MAX;
        cpu_base->softirq_activated = false;
        cpu_base->online = true;
        return 0;
}

#ifdef CONFIG_HOTPLUG_CPU

static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
                                struct hrtimer_clock_base *new_base)
{
        struct timerqueue_linked_node *node;
        struct hrtimer *timer;

        while ((node = timerqueue_linked_first(&old_base->active))) {
                timer = hrtimer_from_timerqueue_node(node);
                BUG_ON(hrtimer_callback_running(timer));
                debug_hrtimer_deactivate(timer);

                /*
                 * Mark it as ENQUEUED not INACTIVE otherwise the
                 * timer could be seen as !active and just vanish away
                 * under us on another CPU
                 */
                __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, false);
                timer->base = new_base;
                /*
                 * Enqueue the timers on the new cpu. This does not
                 * reprogram the event device in case the timer
                 * expires before the earliest on this CPU, but we run
                 * hrtimer_interrupt after we migrated everything to
                 * sort out already expired timers and reprogram the
                 * event device.
                 */
                enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS, true);
        }
}

int hrtimers_cpu_dying(unsigned int dying_cpu)
{
        int ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER));
        struct hrtimer_cpu_base *old_base, *new_base;

        old_base = this_cpu_ptr(&hrtimer_bases);
        new_base = &per_cpu(hrtimer_bases, ncpu);

        /*
         * The caller is globally serialized and nobody else
         * takes two locks at once, deadlock is not possible.
         */
        raw_spin_lock(&old_base->lock);
        raw_spin_lock_nested(&new_base->lock, SINGLE_DEPTH_NESTING);

        for (int i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
                migrate_hrtimer_list(&old_base->clock_base[i], &new_base->clock_base[i]);

        /* Tell the other CPU to retrigger the next event */
        smp_call_function_single(ncpu, retrigger_next_event, NULL, 0);

        raw_spin_unlock(&new_base->lock);
        old_base->online = false;
        raw_spin_unlock(&old_base->lock);

        return 0;
}

#endif /* CONFIG_HOTPLUG_CPU */

void __init hrtimers_init(void)
{
        hrtimers_prepare_cpu(smp_processor_id());
        hrtimers_cpu_starting(smp_processor_id());
        open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
}















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   30 






































































































































































































   30 
















   30 















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
// SPDX-License-Identifier: GPL-2.0-or-later
/* client.c: NFS client sharing and management code
 *
 * Copyright (C) 2006 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */


#include <linux/module.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/time.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/unistd.h>
#include <linux/sunrpc/addr.h>
#include <linux/sunrpc/clnt.h>
#include <linux/sunrpc/stats.h>
#include <linux/sunrpc/metrics.h>
#include <linux/sunrpc/xprtsock.h>
#include <linux/sunrpc/xprtrdma.h>
#include <linux/nfs_fs.h>
#include <linux/nfs_mount.h>
#include <linux/nfs4_mount.h>
#include <linux/lockd/bind.h>
#include <linux/seq_file.h>
#include <linux/mount.h>
#include <linux/vfs.h>
#include <linux/inet.h>
#include <linux/in6.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <net/ipv6.h>
#include <linux/nfs_xdr.h>
#include <linux/sunrpc/bc_xprt.h>
#include <linux/nsproxy.h>
#include <linux/pid_namespace.h>
#include <linux/nfslocalio.h>

#include "nfs4_fs.h"
#include "callback.h"
#include "delegation.h"
#include "iostat.h"
#include "internal.h"
#include "fscache.h"
#include "pnfs.h"
#include "nfs.h"
#include "netns.h"
#include "sysfs.h"
#include "nfs42.h"

#define NFSDBG_FACILITY                NFSDBG_CLIENT

static DECLARE_WAIT_QUEUE_HEAD(nfs_client_active_wq);
static DEFINE_RWLOCK(nfs_version_lock);

static struct nfs_subversion *nfs_version_mods[5] = {
        [2] = NULL,
        [3] = NULL,
        [4] = NULL,
};

/*
 * RPC cruft for NFS
 */
static const struct rpc_version *nfs_version[5] = {
        [2] = NULL,
        [3] = NULL,
        [4] = NULL,
};

const struct rpc_program nfs_program = {
        .name                        = "nfs",
        .number                        = NFS_PROGRAM,
        .nrvers                        = ARRAY_SIZE(nfs_version),
        .version                = nfs_version,
        .pipe_dir_name                = NFS_PIPE_DIRNAME,
};

static struct nfs_subversion *__find_nfs_version(unsigned int version)
{
        struct nfs_subversion *nfs;

        read_lock(&nfs_version_lock);
        nfs = nfs_version_mods[version];
        read_unlock(&nfs_version_lock);
        return nfs;
}

struct nfs_subversion *find_nfs_version(unsigned int version)
{
        struct nfs_subversion *nfs = __find_nfs_version(version);

        if (!nfs && request_module("nfsv%d", version) == 0)
                nfs = __find_nfs_version(version);

        if (!nfs)
                return ERR_PTR(-EPROTONOSUPPORT);

        if (!get_nfs_version(nfs))
                return ERR_PTR(-EAGAIN);

        return nfs;
}

int get_nfs_version(struct nfs_subversion *nfs)
{
        return try_module_get(nfs->owner);
}
EXPORT_SYMBOL_GPL(get_nfs_version);

void put_nfs_version(struct nfs_subversion *nfs)
{
        module_put(nfs->owner);
}

void register_nfs_version(struct nfs_subversion *nfs)
{
        write_lock(&nfs_version_lock);

        nfs_version_mods[nfs->rpc_ops->version] = nfs;
        nfs_version[nfs->rpc_ops->version] = nfs->rpc_vers;

        write_unlock(&nfs_version_lock);
}
EXPORT_SYMBOL_GPL(register_nfs_version);

void unregister_nfs_version(struct nfs_subversion *nfs)
{
        write_lock(&nfs_version_lock);

        nfs_version[nfs->rpc_ops->version] = NULL;
        nfs_version_mods[nfs->rpc_ops->version] = NULL;

        write_unlock(&nfs_version_lock);
}
EXPORT_SYMBOL_GPL(unregister_nfs_version);

/*
 * Allocate a shared client record
 *
 * Since these are allocated/deallocated very rarely, we don't
 * bother putting them in a slab cache...
 */
struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_init)
{
        struct nfs_client *clp;
        int err = -ENOMEM;

        if ((clp = kzalloc_obj(*clp)) == NULL)
                goto error_0;

        clp->cl_minorversion = cl_init->minorversion;
        clp->cl_nfs_mod = cl_init->nfs_mod;
        if (!get_nfs_version(clp->cl_nfs_mod))
                goto error_dealloc;

        clp->rpc_ops = clp->cl_nfs_mod->rpc_ops;

        refcount_set(&clp->cl_count, 1);
        clp->cl_cons_state = NFS_CS_INITING;

        memcpy(&clp->cl_addr, cl_init->addr, cl_init->addrlen);
        clp->cl_addrlen = cl_init->addrlen;

        if (cl_init->hostname) {
                err = -ENOMEM;
                clp->cl_hostname = kstrdup(cl_init->hostname, GFP_KERNEL);
                if (!clp->cl_hostname)
                        goto error_cleanup;
        }

        INIT_LIST_HEAD(&clp->cl_superblocks);
        clp->cl_rpcclient = ERR_PTR(-EINVAL);

        clp->cl_flags = cl_init->init_flags;
        clp->cl_proto = cl_init->proto;
        clp->cl_nconnect = cl_init->nconnect;
        clp->cl_max_connect = cl_init->max_connect ? cl_init->max_connect : 1;
        clp->cl_net = get_net_track(cl_init->net, &clp->cl_ns_tracker, GFP_KERNEL);

#if IS_ENABLED(CONFIG_NFS_LOCALIO)
        seqlock_init(&clp->cl_boot_lock);
        ktime_get_real_ts64(&clp->cl_nfssvc_boot);
        nfs_uuid_init(&clp->cl_uuid);
        INIT_WORK(&clp->cl_local_probe_work, nfs_local_probe_async_work);
#endif /* CONFIG_NFS_LOCALIO */

        clp->cl_principal = "*";
        clp->cl_xprtsec = cl_init->xprtsec;
        return clp;

error_cleanup:
        put_nfs_version(clp->cl_nfs_mod);
error_dealloc:
        kfree(clp);
error_0:
        return ERR_PTR(err);
}
EXPORT_SYMBOL_GPL(nfs_alloc_client);

#if IS_ENABLED(CONFIG_NFS_V4)
static void nfs_cleanup_cb_ident_idr(struct net *net)
{
        struct nfs_net *nn = net_generic(net, nfs_net_id);

        idr_destroy(&nn->cb_ident_idr);
}

/* nfs_client_lock held */
static void nfs_cb_idr_remove_locked(struct nfs_client *clp)
{
        struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id);

        if (clp->cl_cb_ident)
                idr_remove(&nn->cb_ident_idr, clp->cl_cb_ident);
}

static void pnfs_init_server(struct nfs_server *server)
{
        rpc_init_wait_queue(&server->roc_rpcwaitq, "pNFS ROC");
}

#else
static void nfs_cleanup_cb_ident_idr(struct net *net)
{
}

static void nfs_cb_idr_remove_locked(struct nfs_client *clp)
{
}

static void pnfs_init_server(struct nfs_server *server)
{
}

#endif /* CONFIG_NFS_V4 */

/*
 * Destroy a shared client record
 */
void nfs_free_client(struct nfs_client *clp)
{
        nfs_localio_disable_client(clp);

        /* -EIO all pending I/O */
        if (!IS_ERR(clp->cl_rpcclient))
                rpc_shutdown_client(clp->cl_rpcclient);

        put_net_track(clp->cl_net, &clp->cl_ns_tracker);
        put_nfs_version(clp->cl_nfs_mod);
        kfree(clp->cl_hostname);
        kfree(clp->cl_acceptor);
        kfree_rcu(clp, rcu);
}
EXPORT_SYMBOL_GPL(nfs_free_client);

/*
 * Release a reference to a shared client record
 */
void nfs_put_client(struct nfs_client *clp)
{
        struct nfs_net *nn;

        if (!clp)
                return;

        nn = net_generic(clp->cl_net, nfs_net_id);

        if (refcount_dec_and_lock(&clp->cl_count, &nn->nfs_client_lock)) {
                list_del(&clp->cl_share_link);
                nfs_cb_idr_remove_locked(clp);
                spin_unlock(&nn->nfs_client_lock);

                WARN_ON_ONCE(!list_empty(&clp->cl_superblocks));

                clp->rpc_ops->free_client(clp);
        }
}
EXPORT_SYMBOL_GPL(nfs_put_client);

/*
 * Find an nfs_client on the list that matches the initialisation data
 * that is supplied.
 */
static struct nfs_client *nfs_match_client(const struct nfs_client_initdata *data)
{
        struct nfs_client *clp;
        const struct sockaddr *sap = (struct sockaddr *)data->addr;
        struct nfs_net *nn = net_generic(data->net, nfs_net_id);
        int error;

again:
        list_for_each_entry(clp, &nn->nfs_client_list, cl_share_link) {
                const struct sockaddr *clap = (struct sockaddr *)&clp->cl_addr;
                /* Don't match clients that failed to initialise properly */
                if (clp->cl_cons_state < 0)
                        continue;

                /* If a client is still initializing then we need to wait */
                if (clp->cl_cons_state > NFS_CS_READY) {
                        refcount_inc(&clp->cl_count);
                        spin_unlock(&nn->nfs_client_lock);
                        error = nfs_wait_client_init_complete(clp);
                        nfs_put_client(clp);
                        spin_lock(&nn->nfs_client_lock);
                        if (error < 0)
                                return ERR_PTR(error);
                        goto again;
                }

                /* Different NFS versions cannot share the same nfs_client */
                if (clp->rpc_ops != data->nfs_mod->rpc_ops)
                        continue;

                if (clp->cl_proto != data->proto)
                        continue;
                /* Match nfsv4 minorversion */
                if (clp->cl_minorversion != data->minorversion)
                        continue;

                /* Match request for a dedicated DS */
                if (test_bit(NFS_CS_DS, &data->init_flags) !=
                    test_bit(NFS_CS_DS, &clp->cl_flags))
                        continue;

                /* Match the full socket address */
                if (!rpc_cmp_addr_port(sap, clap))
                        /* Match all xprt_switch full socket addresses */
                        if (IS_ERR(clp->cl_rpcclient) ||
                            !rpc_clnt_xprt_switch_has_addr(clp->cl_rpcclient,
                                                           sap))
                                continue;

                /* Match the xprt security policy */
                if (clp->cl_xprtsec.policy != data->xprtsec.policy)
                        continue;
                if (clp->cl_xprtsec.policy == RPC_XPRTSEC_TLS_X509) {
                        if (clp->cl_xprtsec.cert_serial !=
                            data->xprtsec.cert_serial)
                                continue;
                        if (clp->cl_xprtsec.privkey_serial !=
                            data->xprtsec.privkey_serial)
                                continue;
                }

                refcount_inc(&clp->cl_count);
                return clp;
        }
        return NULL;
}

/*
 * Return true if @clp is done initializing, false if still working on it.
 *
 * Use nfs_client_init_status to check if it was successful.
 */
bool nfs_client_init_is_complete(const struct nfs_client *clp)
{
        return clp->cl_cons_state <= NFS_CS_READY;
}
EXPORT_SYMBOL_GPL(nfs_client_init_is_complete);

/*
 * Return 0 if @clp was successfully initialized, -errno otherwise.
 *
 * This must be called *after* nfs_client_init_is_complete() returns true,
 * otherwise it will pop WARN_ON_ONCE and return -EINVAL
 */
int nfs_client_init_status(const struct nfs_client *clp)
{
        /* called without checking nfs_client_init_is_complete */
        if (clp->cl_cons_state > NFS_CS_READY) {
                WARN_ON_ONCE(1);
                return -EINVAL;
        }
        return clp->cl_cons_state;
}
EXPORT_SYMBOL_GPL(nfs_client_init_status);

int nfs_wait_client_init_complete(const struct nfs_client *clp)
{
        return wait_event_killable(nfs_client_active_wq,
                        nfs_client_init_is_complete(clp));
}
EXPORT_SYMBOL_GPL(nfs_wait_client_init_complete);

/*
 * Found an existing client.  Make sure it's ready before returning.
 */
static struct nfs_client *
nfs_found_client(const struct nfs_client_initdata *cl_init,
                 struct nfs_client *clp)
{
        int error;

        error = nfs_wait_client_init_complete(clp);
        if (error < 0) {
                nfs_put_client(clp);
                return ERR_PTR(-ERESTARTSYS);
        }

        if (clp->cl_cons_state < NFS_CS_READY) {
                error = clp->cl_cons_state;
                nfs_put_client(clp);
                return ERR_PTR(error);
        }

        smp_rmb();
        return clp;
}

/*
 * Look up a client by IP address and protocol version
 * - creates a new record if one doesn't yet exist
 */
struct nfs_client *nfs_get_client(const struct nfs_client_initdata *cl_init)
{
        struct nfs_client *clp, *new = NULL;
        struct nfs_net *nn = net_generic(cl_init->net, nfs_net_id);
        const struct nfs_rpc_ops *rpc_ops = cl_init->nfs_mod->rpc_ops;

        if (cl_init->hostname == NULL) {
                WARN_ON(1);
                return ERR_PTR(-EINVAL);
        }

        /* see if the client already exists */
        do {
                spin_lock(&nn->nfs_client_lock);

                clp = nfs_match_client(cl_init);
                if (clp) {
                        spin_unlock(&nn->nfs_client_lock);
                        if (new)
                                new->rpc_ops->free_client(new);
                        if (IS_ERR(clp))
                                return clp;
                        return nfs_found_client(cl_init, clp);
                }
                if (new) {
                        list_add_tail(&new->cl_share_link,
                                        &nn->nfs_client_list);
                        spin_unlock(&nn->nfs_client_lock);
                        new = rpc_ops->init_client(new, cl_init);
                        if (!IS_ERR(new))
                                 nfs_local_probe_async(new);
                        return new;
                }

                spin_unlock(&nn->nfs_client_lock);

                new = rpc_ops->alloc_client(cl_init);
        } while (!IS_ERR(new));

        return new;
}
EXPORT_SYMBOL_GPL(nfs_get_client);

/*
 * Mark a server as ready or failed
 */
void nfs_mark_client_ready(struct nfs_client *clp, int state)
{
        smp_wmb();
        clp->cl_cons_state = state;
        wake_up_all(&nfs_client_active_wq);
}
EXPORT_SYMBOL_GPL(nfs_mark_client_ready);

/*
 * Initialise the timeout values for a connection
 */
void nfs_init_timeout_values(struct rpc_timeout *to, int proto,
                                    int timeo, int retrans)
{
        to->to_initval = timeo * HZ / 10;
        to->to_retries = retrans;

        switch (proto) {
        case XPRT_TRANSPORT_TCP:
        case XPRT_TRANSPORT_TCP_TLS:
        case XPRT_TRANSPORT_RDMA:
                if (retrans == NFS_UNSPEC_RETRANS)
                        to->to_retries = NFS_DEF_TCP_RETRANS;
                if (timeo == NFS_UNSPEC_TIMEO || to->to_initval == 0)
                        to->to_initval = NFS_DEF_TCP_TIMEO * HZ / 10;
                if (to->to_initval > NFS_MAX_TCP_TIMEOUT)
                        to->to_initval = NFS_MAX_TCP_TIMEOUT;
                to->to_increment = to->to_initval;
                to->to_maxval = to->to_initval + (to->to_increment * to->to_retries);
                if (to->to_maxval > NFS_MAX_TCP_TIMEOUT)
                        to->to_maxval = NFS_MAX_TCP_TIMEOUT;
                if (to->to_maxval < to->to_initval)
                        to->to_maxval = to->to_initval;
                to->to_exponential = 0;
                break;
        case XPRT_TRANSPORT_UDP:
                if (retrans == NFS_UNSPEC_RETRANS)
                        to->to_retries = NFS_DEF_UDP_RETRANS;
                if (timeo == NFS_UNSPEC_TIMEO || to->to_initval == 0)
                        to->to_initval = NFS_DEF_UDP_TIMEO * HZ / 10;
                if (to->to_initval > NFS_MAX_UDP_TIMEOUT)
                        to->to_initval = NFS_MAX_UDP_TIMEOUT;
                to->to_maxval = NFS_MAX_UDP_TIMEOUT;
                to->to_exponential = 1;
                break;
        default:
                BUG();
        }
}
EXPORT_SYMBOL_GPL(nfs_init_timeout_values);

/*
 * Create an RPC client handle
 */
int nfs_create_rpc_client(struct nfs_client *clp,
                          const struct nfs_client_initdata *cl_init,
                          rpc_authflavor_t flavor)
{
        struct nfs_net                *nn = net_generic(clp->cl_net, nfs_net_id);
        struct rpc_clnt                *clnt = NULL;
        struct rpc_create_args args = {
                .net                = clp->cl_net,
                .protocol        = clp->cl_proto,
                .nconnect        = clp->cl_nconnect,
                .address        = (struct sockaddr *)&clp->cl_addr,
                .addrsize        = clp->cl_addrlen,
                .timeout        = cl_init->timeparms,
                .servername        = clp->cl_hostname,
                .nodename        = cl_init->nodename,
                .program        = &nfs_program,
                .stats                = &nn->rpcstats,
                .version        = clp->rpc_ops->version,
                .authflavor        = flavor,
                .cred                = cl_init->cred,
                .xprtsec        = cl_init->xprtsec,
                .connect_timeout = cl_init->connect_timeout,
                .reconnect_timeout = cl_init->reconnect_timeout,
        };

        if (test_bit(NFS_CS_DISCRTRY, &clp->cl_flags))
                args.flags |= RPC_CLNT_CREATE_DISCRTRY;
        if (test_bit(NFS_CS_NO_RETRANS_TIMEOUT, &clp->cl_flags))
                args.flags |= RPC_CLNT_CREATE_NO_RETRANS_TIMEOUT;
        if (test_bit(NFS_CS_NORESVPORT, &clp->cl_flags))
                args.flags |= RPC_CLNT_CREATE_NONPRIVPORT;
        if (test_bit(NFS_CS_INFINITE_SLOTS, &clp->cl_flags))
                args.flags |= RPC_CLNT_CREATE_INFINITE_SLOTS;
        if (test_bit(NFS_CS_NOPING, &clp->cl_flags))
                args.flags |= RPC_CLNT_CREATE_NOPING;
        if (test_bit(NFS_CS_REUSEPORT, &clp->cl_flags))
                args.flags |= RPC_CLNT_CREATE_REUSEPORT;
        if (test_bit(NFS_CS_NETUNREACH_FATAL, &clp->cl_flags))
                args.flags |= RPC_CLNT_CREATE_NETUNREACH_FATAL;

        if (!IS_ERR(clp->cl_rpcclient))
                return 0;

        clnt = rpc_create(&args);
        if (IS_ERR(clnt)) {
                dprintk("%s: cannot create RPC client. Error = %ld\n",
                                __func__, PTR_ERR(clnt));
                return PTR_ERR(clnt);
        }

        clnt->cl_principal = clp->cl_principal;
        clp->cl_rpcclient = clnt;
        clnt->cl_max_connect = clp->cl_max_connect;
        return 0;
}
EXPORT_SYMBOL_GPL(nfs_create_rpc_client);

/*
 * Version 2 or 3 client destruction
 */
static void nfs_destroy_server(struct nfs_server *server)
{
        if (server->nlm_host)
                nlmclnt_done(server->nlm_host);
}

/*
 * Version 2 or 3 lockd setup
 */
static int nfs_start_lockd(struct nfs_server *server)
{
        struct nlm_host *host;
        struct nfs_client *clp = server->nfs_client;
        struct nlmclnt_initdata nlm_init = {
                .hostname        = clp->cl_hostname,
                .address        = (struct sockaddr *)&clp->cl_addr,
                .addrlen        = clp->cl_addrlen,
                .nfs_version        = clp->rpc_ops->version,
                .noresvport        = server->flags & NFS_MOUNT_NORESVPORT ?
                                        1 : 0,
                .net                = clp->cl_net,
                .nlmclnt_ops         = clp->cl_nfs_mod->rpc_ops->nlmclnt_ops,
                .cred                = server->cred,
        };

        if (nlm_init.nfs_version > 3)
                return 0;
        if ((server->flags & NFS_MOUNT_LOCAL_FLOCK) &&
                        (server->flags & NFS_MOUNT_LOCAL_FCNTL))
                return 0;

        switch (clp->cl_proto) {
                default:
                        nlm_init.protocol = IPPROTO_TCP;
                        break;
#ifndef CONFIG_NFS_DISABLE_UDP_SUPPORT
                case XPRT_TRANSPORT_UDP:
                        nlm_init.protocol = IPPROTO_UDP;
#endif
        }

        host = nlmclnt_init(&nlm_init);
        if (IS_ERR(host))
                return PTR_ERR(host);

        server->nlm_host = host;
        server->destroy = nfs_destroy_server;
        nfs_sysfs_link_rpc_client(server, nlmclnt_rpc_clnt(host), NULL);
        return 0;
}

/*
 * Create a general RPC client
 */
int nfs_init_server_rpcclient(struct nfs_server *server,
                const struct rpc_timeout *timeo,
                rpc_authflavor_t pseudoflavour)
{
        struct nfs_client *clp = server->nfs_client;

        server->client = rpc_clone_client_set_auth(clp->cl_rpcclient,
                                                        pseudoflavour);
        if (IS_ERR(server->client)) {
                dprintk("%s: couldn't create rpc_client!\n", __func__);
                return PTR_ERR(server->client);
        }

        memcpy(&server->client->cl_timeout_default,
                        timeo,
                        sizeof(server->client->cl_timeout_default));
        server->client->cl_timeout = &server->client->cl_timeout_default;
        server->client->cl_softrtry = 0;
        if (server->flags & NFS_MOUNT_SOFTERR)
                server->client->cl_softerr = 1;
        if (server->flags & NFS_MOUNT_SOFT)
                server->client->cl_softrtry = 1;

        nfs_sysfs_link_rpc_client(server, server->client, NULL);
        return 0;
}
EXPORT_SYMBOL_GPL(nfs_init_server_rpcclient);

/**
 * nfs_init_client - Initialise an NFS2 or NFS3 client
 *
 * @clp: nfs_client to initialise
 * @cl_init: Initialisation parameters
 *
 * Returns pointer to an NFS client, or an ERR_PTR value.
 */
struct nfs_client *nfs_init_client(struct nfs_client *clp,
                                   const struct nfs_client_initdata *cl_init)
{
        int error;

        /* the client is already initialised */
        if (clp->cl_cons_state == NFS_CS_READY)
                return clp;

        /*
         * Create a client RPC handle for doing FSSTAT with UNIX auth only
         * - RFC 2623, sec 2.3.2
         */
        error = nfs_create_rpc_client(clp, cl_init, RPC_AUTH_UNIX);
        nfs_mark_client_ready(clp, error == 0 ? NFS_CS_READY : error);
        if (error < 0) {
                nfs_put_client(clp);
                clp = ERR_PTR(error);
        }
        return clp;
}
EXPORT_SYMBOL_GPL(nfs_init_client);

static void nfs4_server_set_init_caps(struct nfs_server *server)
{
#if IS_ENABLED(CONFIG_NFS_V4)
        /* Set the basic capabilities */
        server->caps = server->nfs_client->cl_mvops->init_caps;
        if (server->flags & NFS_MOUNT_NORDIRPLUS)
                server->caps &= ~NFS_CAP_READDIRPLUS;
        if (server->nfs_client->cl_proto == XPRT_TRANSPORT_RDMA)
                server->caps &= ~NFS_CAP_READ_PLUS;

        /*
         * Don't use NFS uid/gid mapping if we're using AUTH_SYS or lower
         * authentication.
         */
        if (nfs4_disable_idmapping &&
            server->client->cl_auth->au_flavor == RPC_AUTH_UNIX)
                server->caps |= NFS_CAP_UIDGID_NOMAP;
#endif
}

void nfs_server_set_init_caps(struct nfs_server *server)
{
        switch (server->nfs_client->rpc_ops->version) {
        case 2:
                server->caps = NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS;
                break;
        case 3:
                server->caps = NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS;
                if (!(server->flags & NFS_MOUNT_NORDIRPLUS))
                        server->caps |= NFS_CAP_READDIRPLUS;
                break;
        default:
                nfs4_server_set_init_caps(server);
                break;
        }
}
EXPORT_SYMBOL_GPL(nfs_server_set_init_caps);

/*
 * Create a version 2 or 3 client
 */
static int nfs_init_server(struct nfs_server *server,
                           const struct fs_context *fc)
{
        const struct nfs_fs_context *ctx = nfs_fc2context(fc);
        struct rpc_timeout timeparms;
        struct nfs_client_initdata cl_init = {
                .hostname = ctx->nfs_server.hostname,
                .addr = &ctx->nfs_server._address,
                .addrlen = ctx->nfs_server.addrlen,
                .nfs_mod = ctx->nfs_mod,
                .proto = ctx->nfs_server.protocol,
                .net = fc->net_ns,
                .timeparms = &timeparms,
                .cred = server->cred,
                .nconnect = ctx->nfs_server.nconnect,
                .init_flags = (1UL << NFS_CS_REUSEPORT),
                .xprtsec = ctx->xprtsec,
        };
        struct nfs_client *clp;
        int error;

        nfs_init_timeout_values(&timeparms, ctx->nfs_server.protocol,
                                ctx->timeo, ctx->retrans);
        if (ctx->flags & NFS_MOUNT_NORESVPORT)
                set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags);

        if (ctx->flags & NFS_MOUNT_NETUNREACH_FATAL)
                __set_bit(NFS_CS_NETUNREACH_FATAL, &cl_init.init_flags);

        /* Allocate or find a client reference we can use */
        clp = nfs_get_client(&cl_init);
        if (IS_ERR(clp))
                return PTR_ERR(clp);

        server->nfs_client = clp;
        nfs_sysfs_add_server(server);
        nfs_sysfs_link_rpc_client(server, clp->cl_rpcclient, "_state");

        /* Initialise the client representation from the mount data */
        server->flags = ctx->flags;
        server->options = ctx->options;

        switch (clp->rpc_ops->version) {
        case 2:
                server->fattr_valid = NFS_ATTR_FATTR_V2;
                break;
        case 3:
                server->fattr_valid = NFS_ATTR_FATTR_V3;
                break;
        default:
                server->fattr_valid = NFS_ATTR_FATTR_V4;
        }

        if (ctx->bsize) {
                server->bsize = ctx->bsize;
                server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_BSIZE;
        }
        if (ctx->rsize) {
                server->rsize = nfs_io_size(ctx->rsize, clp->cl_proto);
                server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_RSIZE;
        }
        if (ctx->wsize) {
                server->wsize = nfs_io_size(ctx->wsize, clp->cl_proto);
                server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_WSIZE;
        }

        server->acregmin = ctx->acregmin * HZ;
        server->acregmax = ctx->acregmax * HZ;
        server->acdirmin = ctx->acdirmin * HZ;
        server->acdirmax = ctx->acdirmax * HZ;

        /* Start lockd here, before we might error out */
        error = nfs_start_lockd(server);
        if (error < 0)
                goto error;

        server->port = ctx->nfs_server.port;
        server->auth_info = ctx->auth_info;

        error = nfs_init_server_rpcclient(server, &timeparms,
                                          ctx->selected_flavor);
        if (error < 0)
                goto error;

        nfs_server_set_init_caps(server);

        /* Preserve the values of mount_server-related mount options */
        if (ctx->mount_server.addrlen) {
                memcpy(&server->mountd_address, &ctx->mount_server.address,
                        ctx->mount_server.addrlen);
                server->mountd_addrlen = ctx->mount_server.addrlen;
        }
        server->mountd_version = ctx->mount_server.version;
        server->mountd_port = ctx->mount_server.port;
        server->mountd_protocol = ctx->mount_server.protocol;

        server->namelen  = ctx->namlen;
        return 0;

error:
        server->nfs_client = NULL;
        nfs_put_client(clp);
        return error;
}

/*
 * Load up the server record from information gained in an fsinfo record
 */
static void nfs_server_set_fsinfo(struct nfs_server *server,
                                  struct nfs_fsinfo *fsinfo)
{
        struct nfs_client *clp = server->nfs_client;
        unsigned long max_rpc_payload, raw_max_rpc_payload;

        /* Work out a lot of parameters */
        if (server->rsize == 0)
                server->rsize = nfs_io_size(fsinfo->rtpref, clp->cl_proto);
        if (server->wsize == 0)
                server->wsize = nfs_io_size(fsinfo->wtpref, clp->cl_proto);

        if (fsinfo->rtmax >= 512 && server->rsize > fsinfo->rtmax)
                server->rsize = nfs_io_size(fsinfo->rtmax, clp->cl_proto);
        if (fsinfo->wtmax >= 512 && server->wsize > fsinfo->wtmax)
                server->wsize = nfs_io_size(fsinfo->wtmax, clp->cl_proto);

        raw_max_rpc_payload = rpc_max_payload(server->client);
        max_rpc_payload = nfs_block_size(raw_max_rpc_payload, NULL);

        if (server->rsize > max_rpc_payload)
                server->rsize = max_rpc_payload;
        if (server->rsize > NFS_MAX_FILE_IO_SIZE)
                server->rsize = NFS_MAX_FILE_IO_SIZE;
        server->rpages = (server->rsize + PAGE_SIZE - 1) >> PAGE_SHIFT;

        if (server->wsize > max_rpc_payload)
                server->wsize = max_rpc_payload;
        if (server->wsize > NFS_MAX_FILE_IO_SIZE)
                server->wsize = NFS_MAX_FILE_IO_SIZE;

        server->wtmult = nfs_block_bits(fsinfo->wtmult, NULL);

        server->dtsize = nfs_block_size(fsinfo->dtpref, NULL);
        if (server->dtsize > NFS_MAX_FILE_IO_SIZE)
                server->dtsize = NFS_MAX_FILE_IO_SIZE;
        if (server->dtsize > server->rsize)
                server->dtsize = server->rsize;

        if (server->flags & NFS_MOUNT_NOAC) {
                server->acregmin = server->acregmax = 0;
                server->acdirmin = server->acdirmax = 0;
        }

        server->maxfilesize = fsinfo->maxfilesize;

        server->change_attr_type = fsinfo->change_attr_type;

        server->clone_blksize = fsinfo->clone_blksize;
        /* We're airborne Set socket buffersize */
        rpc_setbufsize(server->client, server->wsize + 100, server->rsize + 100);

#ifdef CONFIG_NFS_V4_2
        /*
         * Defaults until limited by the session parameters.
         */
        server->gxasize = min_t(unsigned int, raw_max_rpc_payload,
                                XATTR_SIZE_MAX);
        server->sxasize = min_t(unsigned int, raw_max_rpc_payload,
                                XATTR_SIZE_MAX);
        server->lxasize = min_t(unsigned int, raw_max_rpc_payload,
                                nfs42_listxattr_xdrsize(XATTR_LIST_MAX));

        if (fsinfo->xattr_support)
                server->caps |= NFS_CAP_XATTR;
        else
                server->caps &= ~NFS_CAP_XATTR;
#endif
}

/*
 * Probe filesystem information, including the FSID on v2/v3
 */
static int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *mntfh, struct nfs_fattr *fattr)
{
        struct nfs_fsinfo fsinfo;
        struct nfs_client *clp = server->nfs_client;
        int error;

        if (clp->rpc_ops->set_capabilities != NULL) {
                error = clp->rpc_ops->set_capabilities(server, mntfh);
                if (error < 0)
                        return error;
        }

        fsinfo.fattr = fattr;
        fsinfo.nlayouttypes = 0;
        memset(fsinfo.layouttype, 0, sizeof(fsinfo.layouttype));
        error = clp->rpc_ops->fsinfo(server, mntfh, &fsinfo);
        if (error < 0)
                return error;

        nfs_server_set_fsinfo(server, &fsinfo);

        /* Get some general file system info */
        if (server->namelen == 0) {
                struct nfs_pathconf pathinfo;

                pathinfo.fattr = fattr;
                nfs_fattr_init(fattr);

                if (clp->rpc_ops->pathconf(server, mntfh, &pathinfo) >= 0)
                        server->namelen = pathinfo.max_namelen;
        }

        if (clp->rpc_ops->discover_trunking != NULL &&
                        (server->caps & NFS_CAP_FS_LOCATIONS &&
                         (server->flags & NFS_MOUNT_TRUNK_DISCOVERY))) {
                error = clp->rpc_ops->discover_trunking(server, mntfh);
                if (error < 0)
                        return error;
        }

        return 0;
}

/*
 * Grab the destination's particulars, including lease expiry time.
 *
 * Returns zero if probe succeeded and retrieved FSID matches the FSID
 * we have cached.
 */
int nfs_probe_server(struct nfs_server *server, struct nfs_fh *mntfh)
{
        struct nfs_fattr *fattr;
        int error;

        fattr = nfs_alloc_fattr();
        if (fattr == NULL)
                return -ENOMEM;

        /* Sanity: the probe won't work if the destination server
         * does not recognize the migrated FH. */
        error = nfs_probe_fsinfo(server, mntfh, fattr);

        nfs_free_fattr(fattr);
        return error;
}
EXPORT_SYMBOL_GPL(nfs_probe_server);

/*
 * Copy useful information when duplicating a server record
 */
void nfs_server_copy_userdata(struct nfs_server *target, struct nfs_server *source)
{
        target->flags = source->flags;
        target->automount_inherit = source->automount_inherit;
        if (source->automount_inherit & NFS_AUTOMOUNT_INHERIT_BSIZE)
                target->bsize = source->bsize;
        if (source->automount_inherit & NFS_AUTOMOUNT_INHERIT_RSIZE)
                target->rsize = source->rsize;
        if (source->automount_inherit & NFS_AUTOMOUNT_INHERIT_WSIZE)
                target->wsize = source->wsize;
        target->acregmin = source->acregmin;
        target->acregmax = source->acregmax;
        target->acdirmin = source->acdirmin;
        target->acdirmax = source->acdirmax;
        target->options = source->options;
        target->auth_info = source->auth_info;
        target->port = source->port;
}
EXPORT_SYMBOL_GPL(nfs_server_copy_userdata);

void nfs_server_insert_lists(struct nfs_server *server)
{
        struct nfs_client *clp = server->nfs_client;
        struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id);

        spin_lock(&nn->nfs_client_lock);
        list_add_tail_rcu(&server->client_link, &clp->cl_superblocks);
        list_add_tail(&server->master_link, &nn->nfs_volume_list);
        clear_bit(NFS_CS_STOP_RENEW, &clp->cl_res_state);
        spin_unlock(&nn->nfs_client_lock);

}
EXPORT_SYMBOL_GPL(nfs_server_insert_lists);

void nfs_server_remove_lists(struct nfs_server *server)
{
        struct nfs_client *clp = server->nfs_client;
        struct nfs_net *nn;

        if (clp == NULL)
                return;
        nn = net_generic(clp->cl_net, nfs_net_id);
        spin_lock(&nn->nfs_client_lock);
        list_del_rcu(&server->client_link);
        if (list_empty(&clp->cl_superblocks))
                set_bit(NFS_CS_STOP_RENEW, &clp->cl_res_state);
        list_del(&server->master_link);
        spin_unlock(&nn->nfs_client_lock);

        synchronize_rcu();
}
EXPORT_SYMBOL_GPL(nfs_server_remove_lists);

static DEFINE_IDA(s_sysfs_ids);

/*
 * Allocate and initialise a server record
 */
struct nfs_server *nfs_alloc_server(void)
{
        struct nfs_server *server;

        server = kzalloc_obj(struct nfs_server);
        if (!server)
                return NULL;

        server->s_sysfs_id = ida_alloc(&s_sysfs_ids, GFP_KERNEL);
        if (server->s_sysfs_id < 0) {
                kfree(server);
                return NULL;
        }

        server->client = server->client_acl = ERR_PTR(-EINVAL);

        /* Zero out the NFS state stuff */
        INIT_LIST_HEAD(&server->client_link);
        INIT_LIST_HEAD(&server->master_link);
        INIT_LIST_HEAD(&server->delegations);
        spin_lock_init(&server->delegations_lock);
        INIT_LIST_HEAD(&server->delegations_return);
        INIT_LIST_HEAD(&server->delegations_lru);
        INIT_LIST_HEAD(&server->delegations_delayed);
        INIT_LIST_HEAD(&server->layouts);
        INIT_LIST_HEAD(&server->state_owners_lru);
        INIT_LIST_HEAD(&server->ss_copies);
        INIT_LIST_HEAD(&server->ss_src_copies);

        atomic_set(&server->active, 0);
        atomic_long_set(&server->nr_active_delegations, 0);

        server->io_stats = nfs_alloc_iostats();
        if (!server->io_stats) {
                kfree(server);
                return NULL;
        }

        server->change_attr_type = NFS4_CHANGE_TYPE_IS_UNDEFINED;

        init_waitqueue_head(&server->write_congestion_wait);
        atomic_long_set(&server->writeback, 0);

        atomic64_set(&server->owner_ctr, 0);

        pnfs_init_server(server);
        rpc_init_wait_queue(&server->uoc_rpcwaitq, "NFS UOC");

        return server;
}
EXPORT_SYMBOL_GPL(nfs_alloc_server);

static void delayed_free(struct rcu_head *p)
{
        struct nfs_server *server = container_of(p, struct nfs_server, rcu);

        nfs_free_iostats(server->io_stats);
        kfree(server);
}

/*
 * Free up a server record
 */
void nfs_free_server(struct nfs_server *server)
{
        nfs_server_remove_lists(server);

        if (server->destroy != NULL)
                server->destroy(server);

        if (!IS_ERR(server->client_acl))
                rpc_shutdown_client(server->client_acl);
        if (!IS_ERR(server->client))
                rpc_shutdown_client(server->client);

        nfs_put_client(server->nfs_client);

        if (server->kobj.state_initialized) {
                nfs_sysfs_remove_server(server);
                kobject_put(&server->kobj);
        }
        ida_free(&s_sysfs_ids, server->s_sysfs_id);

        put_cred(server->cred);
        nfs_release_automount_timer();
        call_rcu(&server->rcu, delayed_free);
}
EXPORT_SYMBOL_GPL(nfs_free_server);

/*
 * Create a version 2 or 3 volume record
 * - keyed on server and FSID
 */
struct nfs_server *nfs_create_server(struct fs_context *fc)
{
        struct nfs_fs_context *ctx = nfs_fc2context(fc);
        struct nfs_server *server;
        struct nfs_fattr *fattr;
        int error;

        server = nfs_alloc_server();
        if (!server)
                return ERR_PTR(-ENOMEM);

        server->cred = get_cred(fc->cred);

        error = -ENOMEM;
        fattr = nfs_alloc_fattr();
        if (fattr == NULL)
                goto error;

        /* Get a client representation */
        error = nfs_init_server(server, fc);
        if (error < 0)
                goto error;

        /* Probe the root fh to retrieve its FSID */
        error = nfs_probe_fsinfo(server, ctx->mntfh, fattr);
        if (error < 0)
                goto error;
        if (server->nfs_client->rpc_ops->version == 3) {
                if (server->namelen == 0 || server->namelen > NFS3_MAXNAMLEN)
                        server->namelen = NFS3_MAXNAMLEN;
                if (!(ctx->flags & NFS_MOUNT_NORDIRPLUS))
                        server->caps |= NFS_CAP_READDIRPLUS;
        } else {
                if (server->namelen == 0 || server->namelen > NFS2_MAXNAMLEN)
                        server->namelen = NFS2_MAXNAMLEN;
        }
        /* Linux 'subtree_check' borkenness mandates this setting */
        server->fh_expire_type = NFS_FH_VOL_RENAME;

        if (!(fattr->valid & NFS_ATTR_FATTR)) {
                error = ctx->nfs_mod->rpc_ops->getattr(server, ctx->mntfh,
                                                       fattr, NULL);
                if (error < 0) {
                        dprintk("nfs_create_server: getattr error = %d\n", -error);
                        goto error;
                }
        }
        memcpy(&server->fsid, &fattr->fsid, sizeof(server->fsid));

        dprintk("Server FSID: %llx:%llx\n",
                (unsigned long long) server->fsid.major,
                (unsigned long long) server->fsid.minor);

        nfs_server_insert_lists(server);
        server->mount_time = jiffies;
        nfs_free_fattr(fattr);
        return server;

error:
        nfs_free_fattr(fattr);
        nfs_free_server(server);
        return ERR_PTR(error);
}
EXPORT_SYMBOL_GPL(nfs_create_server);

/*
 * Clone an NFS2, NFS3 or NFS4 server record
 */
struct nfs_server *nfs_clone_server(struct nfs_server *source,
                                    struct nfs_fh *fh,
                                    struct nfs_fattr *fattr,
                                    rpc_authflavor_t flavor)
{
        struct nfs_server *server;
        int error;

        server = nfs_alloc_server();
        if (!server)
                return ERR_PTR(-ENOMEM);

        server->cred = get_cred(source->cred);

        /* Copy data from the source */
        server->nfs_client = source->nfs_client;
        server->destroy = source->destroy;
        refcount_inc(&server->nfs_client->cl_count);
        nfs_server_copy_userdata(server, source);

        server->fsid = fattr->fsid;

        nfs_sysfs_add_server(server);

        nfs_sysfs_link_rpc_client(server,
                server->nfs_client->cl_rpcclient, "_state");

        error = nfs_init_server_rpcclient(server,
                        source->client->cl_timeout,
                        flavor);
        if (error < 0)
                goto out_free_server;

        nfs_server_set_init_caps(server);

        /* probe the filesystem info for this server filesystem */
        error = nfs_probe_server(server, fh);
        if (error < 0)
                goto out_free_server;

        if (server->namelen == 0 || server->namelen > NFS4_MAXNAMLEN)
                server->namelen = NFS4_MAXNAMLEN;

        error = nfs_start_lockd(server);
        if (error < 0)
                goto out_free_server;

        nfs_server_insert_lists(server);
        server->mount_time = jiffies;

        return server;

out_free_server:
        nfs_free_server(server);
        return ERR_PTR(error);
}
EXPORT_SYMBOL_GPL(nfs_clone_server);

void nfs_clients_init(struct net *net)
{
        struct nfs_net *nn = net_generic(net, nfs_net_id);

        INIT_LIST_HEAD(&nn->nfs_client_list);
        INIT_LIST_HEAD(&nn->nfs_volume_list);
#if IS_ENABLED(CONFIG_NFS_V4)
        idr_init(&nn->cb_ident_idr);
        INIT_LIST_HEAD(&nn->nfs4_data_server_cache);
        spin_lock_init(&nn->nfs4_data_server_lock);
#endif /* CONFIG_NFS_V4 */
        spin_lock_init(&nn->nfs_client_lock);
        nn->boot_time = ktime_get_real();
        memset(&nn->rpcstats, 0, sizeof(nn->rpcstats));
        nn->rpcstats.program = &nfs_program;

        nfs_netns_sysfs_setup(nn, net);
}

void nfs_clients_exit(struct net *net)
{
        struct nfs_net *nn = net_generic(net, nfs_net_id);

        nfs_netns_sysfs_destroy(nn);
        nfs_cleanup_cb_ident_idr(net);
        WARN_ON_ONCE(!list_empty(&nn->nfs_client_list));
        WARN_ON_ONCE(!list_empty(&nn->nfs_volume_list));
#if IS_ENABLED(CONFIG_NFS_V4)
        WARN_ON_ONCE(!list_empty(&nn->nfs4_data_server_cache));
#endif /* CONFIG_NFS_V4 */
}

#ifdef CONFIG_PROC_FS
static void *nfs_server_list_start(struct seq_file *p, loff_t *pos);
static void *nfs_server_list_next(struct seq_file *p, void *v, loff_t *pos);
static void nfs_server_list_stop(struct seq_file *p, void *v);
static int nfs_server_list_show(struct seq_file *m, void *v);

static const struct seq_operations nfs_server_list_ops = {
        .start        = nfs_server_list_start,
        .next        = nfs_server_list_next,
        .stop        = nfs_server_list_stop,
        .show        = nfs_server_list_show,
};

static void *nfs_volume_list_start(struct seq_file *p, loff_t *pos);
static void *nfs_volume_list_next(struct seq_file *p, void *v, loff_t *pos);
static void nfs_volume_list_stop(struct seq_file *p, void *v);
static int nfs_volume_list_show(struct seq_file *m, void *v);

static const struct seq_operations nfs_volume_list_ops = {
        .start        = nfs_volume_list_start,
        .next        = nfs_volume_list_next,
        .stop        = nfs_volume_list_stop,
        .show        = nfs_volume_list_show,
};

/*
 * set up the iterator to start reading from the server list and return the first item
 */
static void *nfs_server_list_start(struct seq_file *m, loff_t *_pos)
                                __acquires(&nn->nfs_client_lock)
{
        struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id);

        /* lock the list against modification */
        spin_lock(&nn->nfs_client_lock);
        return seq_list_start_head(&nn->nfs_client_list, *_pos);
}

/*
 * move to next server
 */
static void *nfs_server_list_next(struct seq_file *p, void *v, loff_t *pos)
{
        struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id);

        return seq_list_next(v, &nn->nfs_client_list, pos);
}

/*
 * clean up after reading from the transports list
 */
static void nfs_server_list_stop(struct seq_file *p, void *v)
                                __releases(&nn->nfs_client_lock)
{
        struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id);

        spin_unlock(&nn->nfs_client_lock);
}

/*
 * display a header line followed by a load of call lines
 */
static int nfs_server_list_show(struct seq_file *m, void *v)
{
        struct nfs_client *clp;
        struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id);

        /* display header on line 1 */
        if (v == &nn->nfs_client_list) {
                seq_puts(m, "NV SERVER   PORT USE HOSTNAME\n");
                return 0;
        }

        /* display one transport per line on subsequent lines */
        clp = list_entry(v, struct nfs_client, cl_share_link);

        /* Check if the client is initialized */
        if (clp->cl_cons_state != NFS_CS_READY)
                return 0;

        rcu_read_lock();
        seq_printf(m, "v%u %s %s %3d %s\n",
                   clp->rpc_ops->version,
                   rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_HEX_ADDR),
                   rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_HEX_PORT),
                   refcount_read(&clp->cl_count),
                   clp->cl_hostname);
        rcu_read_unlock();

        return 0;
}

/*
 * set up the iterator to start reading from the volume list and return the first item
 */
static void *nfs_volume_list_start(struct seq_file *m, loff_t *_pos)
                                __acquires(&nn->nfs_client_lock)
{
        struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id);

        /* lock the list against modification */
        spin_lock(&nn->nfs_client_lock);
        return seq_list_start_head(&nn->nfs_volume_list, *_pos);
}

/*
 * move to next volume
 */
static void *nfs_volume_list_next(struct seq_file *p, void *v, loff_t *pos)
{
        struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id);

        return seq_list_next(v, &nn->nfs_volume_list, pos);
}

/*
 * clean up after reading from the transports list
 */
static void nfs_volume_list_stop(struct seq_file *p, void *v)
                                __releases(&nn->nfs_client_lock)
{
        struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id);

        spin_unlock(&nn->nfs_client_lock);
}

/*
 * display a header line followed by a load of call lines
 */
static int nfs_volume_list_show(struct seq_file *m, void *v)
{
        struct nfs_server *server;
        struct nfs_client *clp;
        char dev[13];        // 8 for 2^24, 1 for ':', 3 for 2^8, 1 for '\0'
        char fsid[34];        // 2 * 16 for %llx, 1 for ':', 1 for '\0'
        struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id);

        /* display header on line 1 */
        if (v == &nn->nfs_volume_list) {
                seq_puts(m, "NV SERVER   PORT DEV          FSID"
                            "                              FSC\n");
                return 0;
        }
        /* display one transport per line on subsequent lines */
        server = list_entry(v, struct nfs_server, master_link);
        clp = server->nfs_client;

        snprintf(dev, sizeof(dev), "%u:%u",
                 MAJOR(server->s_dev), MINOR(server->s_dev));

        snprintf(fsid, sizeof(fsid), "%llx:%llx",
                 (unsigned long long) server->fsid.major,
                 (unsigned long long) server->fsid.minor);

        rcu_read_lock();
        seq_printf(m, "v%u %s %s %-12s %-33s %s\n",
                   clp->rpc_ops->version,
                   rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_HEX_ADDR),
                   rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_HEX_PORT),
                   dev,
                   fsid,
                   nfs_server_fscache_state(server));
        rcu_read_unlock();

        return 0;
}

int nfs_fs_proc_net_init(struct net *net)
{
        struct nfs_net *nn = net_generic(net, nfs_net_id);
        struct proc_dir_entry *p;

        nn->proc_nfsfs = proc_net_mkdir(net, "nfsfs", net->proc_net);
        if (!nn->proc_nfsfs)
                goto error_0;

        /* a file of servers with which we're dealing */
        p = proc_create_net("servers", S_IFREG|S_IRUGO, nn->proc_nfsfs,
                        &nfs_server_list_ops, sizeof(struct seq_net_private));
        if (!p)
                goto error_1;

        /* a file of volumes that we have mounted */
        p = proc_create_net("volumes", S_IFREG|S_IRUGO, nn->proc_nfsfs,
                        &nfs_volume_list_ops, sizeof(struct seq_net_private));
        if (!p)
                goto error_1;
        return 0;

error_1:
        remove_proc_subtree("nfsfs", net->proc_net);
error_0:
        return -ENOMEM;
}

void nfs_fs_proc_net_exit(struct net *net)
{
        remove_proc_subtree("nfsfs", net->proc_net);
}

/*
 * initialise the /proc/fs/nfsfs/ directory
 */
int __init nfs_fs_proc_init(void)
{
        if (!proc_mkdir("fs/nfsfs", NULL))
                goto error_0;

        /* a file of servers with which we're dealing */
        if (!proc_symlink("fs/nfsfs/servers", NULL, "../../net/nfsfs/servers"))
                goto error_1;

        /* a file of volumes that we have mounted */
        if (!proc_symlink("fs/nfsfs/volumes", NULL, "../../net/nfsfs/volumes"))
                goto error_1;

        return 0;
error_1:
        remove_proc_subtree("fs/nfsfs", NULL);
error_0:
        return -ENOMEM;
}

/*
 * clean up the /proc/fs/nfsfs/ directory
 */
void nfs_fs_proc_exit(void)
{
        remove_proc_subtree("fs/nfsfs", NULL);
        ida_destroy(&s_sysfs_ids);
}

#endif /* CONFIG_PROC_FS */





























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
 * Author: Joerg Roedel <joerg.roedel@amd.com>
 */

#ifndef __LINUX_IOMMU_H
#define __LINUX_IOMMU_H

#include <linux/scatterlist.h>
#include <linux/device.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/of.h>
#include <linux/iova_bitmap.h>
#include <uapi/linux/iommufd.h>

#define IOMMU_READ        (1 << 0)
#define IOMMU_WRITE        (1 << 1)
#define IOMMU_CACHE        (1 << 2) /* DMA cache coherency */
#define IOMMU_NOEXEC        (1 << 3)
#define IOMMU_MMIO        (1 << 4) /* e.g. things like MSI doorbells */
/*
 * Where the bus hardware includes a privilege level as part of its access type
 * markings, and certain devices are capable of issuing transactions marked as
 * either 'supervisor' or 'user', the IOMMU_PRIV flag requests that the other
 * given permission flags only apply to accesses at the higher privilege level,
 * and that unprivileged transactions should have as little access as possible.
 * This would usually imply the same permissions as kernel mappings on the CPU,
 * if the IOMMU page table format is equivalent.
 */
#define IOMMU_PRIV        (1 << 5)

struct iommu_ops;
struct iommu_group;
struct bus_type;
struct device;
struct iommu_domain;
struct iommu_domain_ops;
struct iommu_dirty_ops;
struct notifier_block;
struct iommu_sva;
struct iommu_dma_cookie;
struct iommu_dma_msi_cookie;
struct iommu_fault_param;
struct iommufd_ctx;
struct iommufd_viommu;
struct msi_desc;
struct msi_msg;

#define IOMMU_FAULT_PERM_READ        (1 << 0) /* read */
#define IOMMU_FAULT_PERM_WRITE        (1 << 1) /* write */
#define IOMMU_FAULT_PERM_EXEC        (1 << 2) /* exec */
#define IOMMU_FAULT_PERM_PRIV        (1 << 3) /* privileged */

/* Generic fault types, can be expanded IRQ remapping fault */
enum iommu_fault_type {
        IOMMU_FAULT_PAGE_REQ = 1,        /* page request fault */
};

/**
 * struct iommu_fault_page_request - Page Request data
 * @flags: encodes whether the corresponding fields are valid and whether this
 *         is the last page in group (IOMMU_FAULT_PAGE_REQUEST_* values).
 *         When IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID is set, the page response
 *         must have the same PASID value as the page request. When it is clear,
 *         the page response should not have a PASID.
 * @pasid: Process Address Space ID
 * @grpid: Page Request Group Index
 * @perm: requested page permissions (IOMMU_FAULT_PERM_* values)
 * @addr: page address
 * @private_data: device-specific private information
 */
struct iommu_fault_page_request {
#define IOMMU_FAULT_PAGE_REQUEST_PASID_VALID        (1 << 0)
#define IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE        (1 << 1)
#define IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID        (1 << 2)
        u32        flags;
        u32        pasid;
        u32        grpid;
        u32        perm;
        u64        addr;
        u64        private_data[2];
};

/**
 * struct iommu_fault - Generic fault data
 * @type: fault type from &enum iommu_fault_type
 * @prm: Page Request message, when @type is %IOMMU_FAULT_PAGE_REQ
 */
struct iommu_fault {
        u32 type;
        struct iommu_fault_page_request prm;
};

/**
 * enum iommu_page_response_code - Return status of fault handlers
 * @IOMMU_PAGE_RESP_SUCCESS: Fault has been handled and the page tables
 *        populated, retry the access. This is "Success" in PCI PRI.
 * @IOMMU_PAGE_RESP_FAILURE: General error. Drop all subsequent faults from
 *        this device if possible. This is "Response Failure" in PCI PRI.
 * @IOMMU_PAGE_RESP_INVALID: Could not handle this fault, don't retry the
 *        access. This is "Invalid Request" in PCI PRI.
 */
enum iommu_page_response_code {
        IOMMU_PAGE_RESP_SUCCESS = 0,
        IOMMU_PAGE_RESP_INVALID,
        IOMMU_PAGE_RESP_FAILURE,
};

/**
 * struct iommu_page_response - Generic page response information
 * @pasid: Process Address Space ID
 * @grpid: Page Request Group Index
 * @code: response code from &enum iommu_page_response_code
 */
struct iommu_page_response {
        u32        pasid;
        u32        grpid;
        u32        code;
};

struct iopf_fault {
        struct iommu_fault fault;
        /* node for pending lists */
        struct list_head list;
};

struct iopf_group {
        struct iopf_fault last_fault;
        struct list_head faults;
        size_t fault_count;
        /* list node for iommu_fault_param::faults */
        struct list_head pending_node;
        struct work_struct work;
        struct iommu_attach_handle *attach_handle;
        /* The device's fault data parameter. */
        struct iommu_fault_param *fault_param;
        /* Used by handler provider to hook the group on its own lists. */
        struct list_head node;
        u32 cookie;
};

/**
 * struct iopf_queue - IO Page Fault queue
 * @wq: the fault workqueue
 * @devices: devices attached to this queue
 * @lock: protects the device list
 */
struct iopf_queue {
        struct workqueue_struct *wq;
        struct list_head devices;
        struct mutex lock;
};

/* iommu fault flags */
#define IOMMU_FAULT_READ        0x0
#define IOMMU_FAULT_WRITE        0x1

typedef int (*iommu_fault_handler_t)(struct iommu_domain *,
                        struct device *, unsigned long, int, void *);

struct iommu_domain_geometry {
        dma_addr_t aperture_start; /* First address that can be mapped    */
        dma_addr_t aperture_end;   /* Last address that can be mapped     */
        bool force_aperture;       /* DMA only allowed in mappable range? */
};

enum iommu_domain_cookie_type {
        IOMMU_COOKIE_NONE,
        IOMMU_COOKIE_DMA_IOVA,
        IOMMU_COOKIE_DMA_MSI,
        IOMMU_COOKIE_FAULT_HANDLER,
        IOMMU_COOKIE_SVA,
        IOMMU_COOKIE_IOMMUFD,
};

/* Domain feature flags */
#define __IOMMU_DOMAIN_PAGING        (1U << 0)  /* Support for iommu_map/unmap */
#define __IOMMU_DOMAIN_DMA_API        (1U << 1)  /* Domain for use in DMA-API
                                              implementation              */
#define __IOMMU_DOMAIN_PT        (1U << 2)  /* Domain is identity mapped   */
#define __IOMMU_DOMAIN_DMA_FQ        (1U << 3)  /* DMA-API uses flush queue    */

#define __IOMMU_DOMAIN_SVA        (1U << 4)  /* Shared process address space */
#define __IOMMU_DOMAIN_PLATFORM        (1U << 5)

#define __IOMMU_DOMAIN_NESTED        (1U << 6)  /* User-managed address space nested
                                              on a stage-2 translation        */

#define IOMMU_DOMAIN_ALLOC_FLAGS ~__IOMMU_DOMAIN_DMA_FQ
/*
 * This are the possible domain-types
 *
 *        IOMMU_DOMAIN_BLOCKED        - All DMA is blocked, can be used to isolate
 *                                  devices
 *        IOMMU_DOMAIN_IDENTITY        - DMA addresses are system physical addresses
 *        IOMMU_DOMAIN_UNMANAGED        - DMA mappings managed by IOMMU-API user, used
 *                                  for VMs
 *        IOMMU_DOMAIN_DMA        - Internally used for DMA-API implementations.
 *                                  This flag allows IOMMU drivers to implement
 *                                  certain optimizations for these domains
 *        IOMMU_DOMAIN_DMA_FQ        - As above, but definitely using batched TLB
 *                                  invalidation.
 *        IOMMU_DOMAIN_SVA        - DMA addresses are shared process addresses
 *                                  represented by mm_struct's.
 *        IOMMU_DOMAIN_PLATFORM        - Legacy domain for drivers that do their own
 *                                  dma_api stuff. Do not use in new drivers.
 */
#define IOMMU_DOMAIN_BLOCKED        (0U)
#define IOMMU_DOMAIN_IDENTITY        (__IOMMU_DOMAIN_PT)
#define IOMMU_DOMAIN_UNMANAGED        (__IOMMU_DOMAIN_PAGING)
#define IOMMU_DOMAIN_DMA        (__IOMMU_DOMAIN_PAGING |        \
                                 __IOMMU_DOMAIN_DMA_API)
#define IOMMU_DOMAIN_DMA_FQ        (__IOMMU_DOMAIN_PAGING |        \
                                 __IOMMU_DOMAIN_DMA_API |        \
                                 __IOMMU_DOMAIN_DMA_FQ)
#define IOMMU_DOMAIN_SVA        (__IOMMU_DOMAIN_SVA)
#define IOMMU_DOMAIN_PLATFORM        (__IOMMU_DOMAIN_PLATFORM)
#define IOMMU_DOMAIN_NESTED        (__IOMMU_DOMAIN_NESTED)

struct iommu_domain {
        unsigned type;
        enum iommu_domain_cookie_type cookie_type;
        bool is_iommupt;
        const struct iommu_domain_ops *ops;
        const struct iommu_dirty_ops *dirty_ops;
        const struct iommu_ops *owner; /* Whose domain_alloc we came from */
        unsigned long pgsize_bitmap;        /* Bitmap of page sizes in use */
        struct iommu_domain_geometry geometry;
        int (*iopf_handler)(struct iopf_group *group);

        union { /* cookie */
                struct iommu_dma_cookie *iova_cookie;
                struct iommu_dma_msi_cookie *msi_cookie;
                struct iommufd_hw_pagetable *iommufd_hwpt;
                struct {
                        iommu_fault_handler_t handler;
                        void *handler_token;
                };
                struct {        /* IOMMU_DOMAIN_SVA */
                        struct mm_struct *mm;
                        int users;
                        /*
                         * Next iommu_domain in mm->iommu_mm->sva-domains list
                         * protected by iommu_sva_lock.
                         */
                        struct list_head next;
                };
        };
};

static inline bool iommu_is_dma_domain(struct iommu_domain *domain)
{
        return domain->type & __IOMMU_DOMAIN_DMA_API;
}

enum iommu_cap {
        IOMMU_CAP_CACHE_COHERENCY,        /* IOMMU_CACHE is supported */
        IOMMU_CAP_NOEXEC,                /* IOMMU_NOEXEC flag */
        IOMMU_CAP_PRE_BOOT_PROTECTION,        /* Firmware says it used the IOMMU for
                                           DMA protection and we should too */
        /*
         * Per-device flag indicating if enforce_cache_coherency() will work on
         * this device.
         */
        IOMMU_CAP_ENFORCE_CACHE_COHERENCY,
        /*
         * IOMMU driver does not issue TLB maintenance during .unmap, so can
         * usefully support the non-strict DMA flush queue.
         */
        IOMMU_CAP_DEFERRED_FLUSH,
        IOMMU_CAP_DIRTY_TRACKING,        /* IOMMU supports dirty tracking */
        /* ATS is supported and may be enabled for this device */
        IOMMU_CAP_PCI_ATS_SUPPORTED,
};

/* These are the possible reserved region types */
enum iommu_resv_type {
        /* Memory regions which must be mapped 1:1 at all times */
        IOMMU_RESV_DIRECT,
        /*
         * Memory regions which are advertised to be 1:1 but are
         * commonly considered relaxable in some conditions,
         * for instance in device assignment use case (USB, Graphics)
         */
        IOMMU_RESV_DIRECT_RELAXABLE,
        /* Arbitrary "never map this or give it to a device" address ranges */
        IOMMU_RESV_RESERVED,
        /* Hardware MSI region (untranslated) */
        IOMMU_RESV_MSI,
        /* Software-managed MSI translation window */
        IOMMU_RESV_SW_MSI,
};

/**
 * struct iommu_resv_region - descriptor for a reserved memory region
 * @list: Linked list pointers
 * @start: System physical start address of the region
 * @length: Length of the region in bytes
 * @prot: IOMMU Protection flags (READ/WRITE/...)
 * @type: Type of the reserved region
 * @free: Callback to free associated memory allocations
 */
struct iommu_resv_region {
        struct list_head        list;
        phys_addr_t                start;
        size_t                        length;
        int                        prot;
        enum iommu_resv_type        type;
        void (*free)(struct device *dev, struct iommu_resv_region *region);
};

struct iommu_iort_rmr_data {
        struct iommu_resv_region rr;

        /* Stream IDs associated with IORT RMR entry */
        const u32 *sids;
        u32 num_sids;
};

#define IOMMU_NO_PASID        (0U) /* Reserved for DMA w/o PASID */
#define IOMMU_FIRST_GLOBAL_PASID        (1U) /*starting range for allocation */
#define IOMMU_PASID_INVALID        (-1U)
typedef unsigned int ioasid_t;

/* Read but do not clear any dirty bits */
#define IOMMU_DIRTY_NO_CLEAR (1 << 0)

/*
 * Pages allocated through iommu_alloc_pages_node_sz() can be placed on this
 * list using iommu_pages_list_add(). Note: ONLY pages from
 * iommu_alloc_pages_node_sz() can be used this way!
 */
struct iommu_pages_list {
        struct list_head pages;
};

#define IOMMU_PAGES_LIST_INIT(name) \
        ((struct iommu_pages_list){ .pages = LIST_HEAD_INIT(name.pages) })

#ifdef CONFIG_IOMMU_API

/**
 * struct iommu_iotlb_gather - Range information for a pending IOTLB flush
 *
 * @start: IOVA representing the start of the range to be flushed
 * @end: IOVA representing the end of the range to be flushed (inclusive)
 * @pgsize: The interval at which to perform the flush
 * @freelist: Removed pages to free after sync
 * @queued: Indicates that the flush will be queued
 *
 * This structure is intended to be updated by multiple calls to the
 * ->unmap() function in struct iommu_ops before eventually being passed
 * into ->iotlb_sync(). Drivers can add pages to @freelist to be freed after
 * ->iotlb_sync() or ->iotlb_flush_all() have cleared all cached references to
 * them. @queued is set to indicate when ->iotlb_flush_all() will be called
 * later instead of ->iotlb_sync(), so drivers may optimise accordingly.
 */
struct iommu_iotlb_gather {
        unsigned long                start;
        unsigned long                end;
        size_t                        pgsize;
        struct iommu_pages_list        freelist;
        bool                        queued;
};

/**
 * struct iommu_dirty_bitmap - Dirty IOVA bitmap state
 * @bitmap: IOVA bitmap
 * @gather: Range information for a pending IOTLB flush
 */
struct iommu_dirty_bitmap {
        struct iova_bitmap *bitmap;
        struct iommu_iotlb_gather *gather;
};

/**
 * struct iommu_dirty_ops - domain specific dirty tracking operations
 * @set_dirty_tracking: Enable or Disable dirty tracking on the iommu domain
 * @read_and_clear_dirty: Walk IOMMU page tables for dirtied PTEs marshalled
 *                        into a bitmap, with a bit represented as a page.
 *                        Reads the dirty PTE bits and clears it from IO
 *                        pagetables.
 */
struct iommu_dirty_ops {
        int (*set_dirty_tracking)(struct iommu_domain *domain, bool enabled);
        int (*read_and_clear_dirty)(struct iommu_domain *domain,
                                    unsigned long iova, size_t size,
                                    unsigned long flags,
                                    struct iommu_dirty_bitmap *dirty);
};

/**
 * struct iommu_user_data - iommu driver specific user space data info
 * @type: The data type of the user buffer
 * @uptr: Pointer to the user buffer for copy_from_user()
 * @len: The length of the user buffer in bytes
 *
 * A user space data is an uAPI that is defined in include/uapi/linux/iommufd.h
 * @type, @uptr and @len should be just copied from an iommufd core uAPI struct.
 */
struct iommu_user_data {
        unsigned int type;
        void __user *uptr;
        size_t len;
};

/**
 * struct iommu_user_data_array - iommu driver specific user space data array
 * @type: The data type of all the entries in the user buffer array
 * @uptr: Pointer to the user buffer array
 * @entry_len: The fixed-width length of an entry in the array, in bytes
 * @entry_num: The number of total entries in the array
 *
 * The user buffer includes an array of requests with format defined in
 * include/uapi/linux/iommufd.h
 */
struct iommu_user_data_array {
        unsigned int type;
        void __user *uptr;
        size_t entry_len;
        u32 entry_num;
};

/**
 * __iommu_copy_struct_from_user - Copy iommu driver specific user space data
 * @dst_data: Pointer to an iommu driver specific user data that is defined in
 *            include/uapi/linux/iommufd.h
 * @src_data: Pointer to a struct iommu_user_data for user space data info
 * @data_type: The data type of the @dst_data. Must match with @src_data.type
 * @data_len: Length of current user data structure, i.e. sizeof(struct _dst)
 * @min_len: Initial length of user data structure for backward compatibility.
 *           This should be offsetofend using the last member in the user data
 *           struct that was initially added to include/uapi/linux/iommufd.h
 */
static inline int __iommu_copy_struct_from_user(
        void *dst_data, const struct iommu_user_data *src_data,
        unsigned int data_type, size_t data_len, size_t min_len)
{
        if (WARN_ON(!dst_data || !src_data))
                return -EINVAL;
        if (src_data->type != data_type)
                return -EINVAL;
        if (src_data->len < min_len || data_len < src_data->len)
                return -EINVAL;
        return copy_struct_from_user(dst_data, data_len, src_data->uptr,
                                     src_data->len);
}

/**
 * iommu_copy_struct_from_user - Copy iommu driver specific user space data
 * @kdst: Pointer to an iommu driver specific user data that is defined in
 *        include/uapi/linux/iommufd.h
 * @user_data: Pointer to a struct iommu_user_data for user space data info
 * @data_type: The data type of the @kdst. Must match with @user_data->type
 * @min_last: The last member of the data structure @kdst points in the initial
 *            version.
 * Return 0 for success, otherwise -error.
 */
#define iommu_copy_struct_from_user(kdst, user_data, data_type, min_last) \
        __iommu_copy_struct_from_user(kdst, user_data, data_type,         \
                                      sizeof(*kdst),                      \
                                      offsetofend(typeof(*kdst), min_last))

/**
 * __iommu_copy_struct_from_user_array - Copy iommu driver specific user space
 *                                       data from an iommu_user_data_array
 * @dst_data: Pointer to an iommu driver specific user data that is defined in
 *            include/uapi/linux/iommufd.h
 * @src_array: Pointer to a struct iommu_user_data_array for a user space array
 * @data_type: The data type of the @dst_data. Must match with @src_array.type
 * @index: Index to the location in the array to copy user data from
 * @data_len: Length of current user data structure, i.e. sizeof(struct _dst)
 * @min_len: Initial length of user data structure for backward compatibility.
 *           This should be offsetofend using the last member in the user data
 *           struct that was initially added to include/uapi/linux/iommufd.h
 */
static inline int __iommu_copy_struct_from_user_array(
        void *dst_data, const struct iommu_user_data_array *src_array,
        unsigned int data_type, unsigned int index, size_t data_len,
        size_t min_len)
{
        struct iommu_user_data src_data;

        if (WARN_ON(!src_array || index >= src_array->entry_num))
                return -EINVAL;
        if (!src_array->entry_num)
                return -EINVAL;
        src_data.uptr = src_array->uptr + src_array->entry_len * index;
        src_data.len = src_array->entry_len;
        src_data.type = src_array->type;

        return __iommu_copy_struct_from_user(dst_data, &src_data, data_type,
                                             data_len, min_len);
}

/**
 * iommu_copy_struct_from_user_array - Copy iommu driver specific user space
 *                                     data from an iommu_user_data_array
 * @kdst: Pointer to an iommu driver specific user data that is defined in
 *        include/uapi/linux/iommufd.h
 * @user_array: Pointer to a struct iommu_user_data_array for a user space
 *              array
 * @data_type: The data type of the @kdst. Must match with @user_array->type
 * @index: Index to the location in the array to copy user data from
 * @min_last: The last member of the data structure @kdst points in the
 *            initial version.
 *
 * Copy a single entry from a user array. Return 0 for success, otherwise
 * -error.
 */
#define iommu_copy_struct_from_user_array(kdst, user_array, data_type, index, \
                                          min_last)                           \
        __iommu_copy_struct_from_user_array(                                  \
                kdst, user_array, data_type, index, sizeof(*(kdst)),          \
                offsetofend(typeof(*(kdst)), min_last))

/**
 * iommu_copy_struct_from_full_user_array - Copy iommu driver specific user
 *         space data from an iommu_user_data_array
 * @kdst: Pointer to an iommu driver specific user data that is defined in
 *        include/uapi/linux/iommufd.h
 * @kdst_entry_size: sizeof(*kdst)
 * @user_array: Pointer to a struct iommu_user_data_array for a user space
 *              array
 * @data_type: The data type of the @kdst. Must match with @user_array->type
 *
 * Copy the entire user array. kdst must have room for kdst_entry_size *
 * user_array->entry_num bytes. Return 0 for success, otherwise -error.
 */
static inline int
iommu_copy_struct_from_full_user_array(void *kdst, size_t kdst_entry_size,
                                       struct iommu_user_data_array *user_array,
                                       unsigned int data_type)
{
        unsigned int i;
        int ret;

        if (user_array->type != data_type)
                return -EINVAL;
        if (!user_array->entry_num)
                return -EINVAL;
        if (likely(user_array->entry_len == kdst_entry_size)) {
                if (copy_from_user(kdst, user_array->uptr,
                                   user_array->entry_num *
                                           user_array->entry_len))
                        return -EFAULT;
        }

        /* Copy item by item */
        for (i = 0; i != user_array->entry_num; i++) {
                ret = copy_struct_from_user(
                        kdst + kdst_entry_size * i, kdst_entry_size,
                        user_array->uptr + user_array->entry_len * i,
                        user_array->entry_len);
                if (ret)
                        return ret;
        }
        return 0;
}

/**
 * __iommu_copy_struct_to_user - Report iommu driver specific user space data
 * @dst_data: Pointer to a struct iommu_user_data for user space data location
 * @src_data: Pointer to an iommu driver specific user data that is defined in
 *            include/uapi/linux/iommufd.h
 * @data_type: The data type of the @src_data. Must match with @dst_data.type
 * @data_len: Length of current user data structure, i.e. sizeof(struct _src)
 * @min_len: Initial length of user data structure for backward compatibility.
 *           This should be offsetofend using the last member in the user data
 *           struct that was initially added to include/uapi/linux/iommufd.h
 */
static inline int
__iommu_copy_struct_to_user(const struct iommu_user_data *dst_data,
                            void *src_data, unsigned int data_type,
                            size_t data_len, size_t min_len)
{
        if (WARN_ON(!dst_data || !src_data))
                return -EINVAL;
        if (dst_data->type != data_type)
                return -EINVAL;
        if (dst_data->len < min_len || data_len < dst_data->len)
                return -EINVAL;
        return copy_struct_to_user(dst_data->uptr, dst_data->len, src_data,
                                   data_len, NULL);
}

/**
 * iommu_copy_struct_to_user - Report iommu driver specific user space data
 * @user_data: Pointer to a struct iommu_user_data for user space data location
 * @ksrc: Pointer to an iommu driver specific user data that is defined in
 *        include/uapi/linux/iommufd.h
 * @data_type: The data type of the @ksrc. Must match with @user_data->type
 * @min_last: The last member of the data structure @ksrc points in the initial
 *            version.
 * Return 0 for success, otherwise -error.
 */
#define iommu_copy_struct_to_user(user_data, ksrc, data_type, min_last)        \
        __iommu_copy_struct_to_user(user_data, ksrc, data_type, sizeof(*ksrc), \
                                    offsetofend(typeof(*ksrc), min_last))

/**
 * struct iommu_ops - iommu ops and capabilities
 * @capable: check capability
 * @hw_info: report iommu hardware information. The data buffer returned by this
 *           op is allocated in the iommu driver and freed by the caller after
 *           use. @type can input a requested type and output a supported type.
 *           Driver should reject an unsupported data @type input
 * @domain_alloc: Do not use in new drivers
 * @domain_alloc_identity: allocate an IDENTITY domain. Drivers should prefer to
 *                         use identity_domain instead. This should only be used
 *                         if dynamic logic is necessary.
 * @domain_alloc_paging_flags: Allocate an iommu domain corresponding to the
 *                     input parameters as defined in
 *                     include/uapi/linux/iommufd.h. The @user_data can be
 *                     optionally provided, the new domain must support
 *                     __IOMMU_DOMAIN_PAGING. Upon failure, ERR_PTR must be
 *                     returned.
 * @domain_alloc_paging: Allocate an iommu_domain that can be used for
 *                       UNMANAGED, DMA, and DMA_FQ domain types. This is the
 *                       same as invoking domain_alloc_paging_flags() with
 *                       @flags=0, @user_data=NULL. A driver should implement
 *                       only one of the two ops.
 * @domain_alloc_sva: Allocate an iommu_domain for Shared Virtual Addressing.
 * @domain_alloc_nested: Allocate an iommu_domain for nested translation.
 * @probe_device: Add device to iommu driver handling
 * @release_device: Remove device from iommu driver handling
 * @probe_finalize: Do final setup work after the device is added to an IOMMU
 *                  group and attached to the groups domain
 * @device_group: find iommu group for a particular device
 * @get_resv_regions: Request list of reserved regions for a device
 * @of_xlate: add OF master IDs to iommu grouping
 * @is_attach_deferred: Check if domain attach should be deferred from iommu
 *                      driver init to device driver init (default no)
 * @page_response: handle page request response
 * @def_domain_type: device default domain type, return value:
 *                - IOMMU_DOMAIN_IDENTITY: must use an identity domain
 *                - IOMMU_DOMAIN_DMA: must use a dma domain
 *                - 0: use the default setting
 * @default_domain_ops: the default ops for domains
 * @get_viommu_size: Get the size of a driver-level vIOMMU structure for a given
 *                   @dev corresponding to @viommu_type. Driver should return 0
 *                   if vIOMMU isn't supported accordingly. It is required for
 *                   driver to use the VIOMMU_STRUCT_SIZE macro to sanitize the
 *                   driver-level vIOMMU structure related to the core one
 * @viommu_init: Init the driver-level struct of an iommufd_viommu on a physical
 *               IOMMU instance @viommu->iommu_dev, as the set of virtualization
 *               resources shared/passed to user space IOMMU instance. Associate
 *               it with a nesting @parent_domain. It is required for driver to
 *               set @viommu->ops pointing to its own viommu_ops
 * @owner: Driver module providing these ops
 * @identity_domain: An always available, always attachable identity
 *                   translation.
 * @blocked_domain: An always available, always attachable blocking
 *                  translation.
 * @default_domain: If not NULL this will always be set as the default domain.
 *                  This should be an IDENTITY/BLOCKED/PLATFORM domain.
 *                  Do not use in new drivers.
 * @user_pasid_table: IOMMU driver supports user-managed PASID table. There is
 *                    no user domain for each PASID and the I/O page faults are
 *                    forwarded through the user domain attached to the device
 *                    RID.
 */
struct iommu_ops {
        bool (*capable)(struct device *dev, enum iommu_cap);
        void *(*hw_info)(struct device *dev, u32 *length,
                         enum iommu_hw_info_type *type);

        /* Domain allocation and freeing by the iommu driver */
#if IS_ENABLED(CONFIG_FSL_PAMU)
        struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type);
#endif
        struct iommu_domain *(*domain_alloc_identity)(struct device *dev);
        struct iommu_domain *(*domain_alloc_paging_flags)(
                struct device *dev, u32 flags,
                const struct iommu_user_data *user_data);
        struct iommu_domain *(*domain_alloc_paging)(struct device *dev);
        struct iommu_domain *(*domain_alloc_sva)(struct device *dev,
                                                 struct mm_struct *mm);
        struct iommu_domain *(*domain_alloc_nested)(
                struct device *dev, struct iommu_domain *parent, u32 flags,
                const struct iommu_user_data *user_data);

        struct iommu_device *(*probe_device)(struct device *dev);
        void (*release_device)(struct device *dev);
        void (*probe_finalize)(struct device *dev);
        struct iommu_group *(*device_group)(struct device *dev);

        /* Request/Free a list of reserved regions for a device */
        void (*get_resv_regions)(struct device *dev, struct list_head *list);

        int (*of_xlate)(struct device *dev, const struct of_phandle_args *args);
        bool (*is_attach_deferred)(struct device *dev);

        /* Per device IOMMU features */
        void (*page_response)(struct device *dev, struct iopf_fault *evt,
                              struct iommu_page_response *msg);

        int (*def_domain_type)(struct device *dev);

        size_t (*get_viommu_size)(struct device *dev,
                                  enum iommu_viommu_type viommu_type);
        int (*viommu_init)(struct iommufd_viommu *viommu,
                           struct iommu_domain *parent_domain,
                           const struct iommu_user_data *user_data);

        const struct iommu_domain_ops *default_domain_ops;
        struct module *owner;
        struct iommu_domain *identity_domain;
        struct iommu_domain *blocked_domain;
        struct iommu_domain *release_domain;
        struct iommu_domain *default_domain;
        u8 user_pasid_table:1;
};

/**
 * struct iommu_domain_ops - domain specific operations
 * @attach_dev: attach an iommu domain to a device
 *  Return:
 * * 0                - success
 * * EINVAL        - can indicate that device and domain are incompatible due to
 *                  some previous configuration of the domain, in which case the
 *                  driver shouldn't log an error, since it is legitimate for a
 *                  caller to test reuse of existing domains. Otherwise, it may
 *                  still represent some other fundamental problem
 * * ENOMEM        - out of memory
 * * ENOSPC        - non-ENOMEM type of resource allocation failures
 * * EBUSY        - device is attached to a domain and cannot be changed
 * * ENODEV        - device specific errors, not able to be attached
 * * <others>        - treated as ENODEV by the caller. Use is discouraged
 * @set_dev_pasid: set or replace an iommu domain to a pasid of device. The pasid of
 *                 the device should be left in the old config in error case.
 * @map_pages: map a physically contiguous set of pages of the same size to
 *             an iommu domain.
 * @unmap_pages: unmap a number of pages of the same size from an iommu domain
 * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
 * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
 * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
 *            queue
 * @cache_invalidate_user: Flush hardware cache for user space IO page table.
 *                         The @domain must be IOMMU_DOMAIN_NESTED. The @array
 *                         passes in the cache invalidation requests, in form
 *                         of a driver data structure. The driver must update
 *                         array->entry_num to report the number of handled
 *                         invalidation requests. The driver data structure
 *                         must be defined in include/uapi/linux/iommufd.h
 * @iova_to_phys: translate iova to physical address
 * @enforce_cache_coherency: Prevent any kind of DMA from bypassing IOMMU_CACHE,
 *                           including no-snoop TLPs on PCIe or other platform
 *                           specific mechanisms.
 * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*)
 * @free: Release the domain after use.
 */
struct iommu_domain_ops {
        int (*attach_dev)(struct iommu_domain *domain, struct device *dev,
                          struct iommu_domain *old);
        int (*set_dev_pasid)(struct iommu_domain *domain, struct device *dev,
                             ioasid_t pasid, struct iommu_domain *old);

        int (*map_pages)(struct iommu_domain *domain, unsigned long iova,
                         phys_addr_t paddr, size_t pgsize, size_t pgcount,
                         int prot, gfp_t gfp, size_t *mapped);
        size_t (*unmap_pages)(struct iommu_domain *domain, unsigned long iova,
                              size_t pgsize, size_t pgcount,
                              struct iommu_iotlb_gather *iotlb_gather);

        void (*flush_iotlb_all)(struct iommu_domain *domain);
        int (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova,
                              size_t size);
        void (*iotlb_sync)(struct iommu_domain *domain,
                           struct iommu_iotlb_gather *iotlb_gather);
        int (*cache_invalidate_user)(struct iommu_domain *domain,
                                     struct iommu_user_data_array *array);

        phys_addr_t (*iova_to_phys)(struct iommu_domain *domain,
                                    dma_addr_t iova);

        bool (*enforce_cache_coherency)(struct iommu_domain *domain);
        int (*set_pgtable_quirks)(struct iommu_domain *domain,
                                  unsigned long quirks);

        void (*free)(struct iommu_domain *domain);
};

/**
 * struct iommu_device - IOMMU core representation of one IOMMU hardware
 *                         instance
 * @list: Used by the iommu-core to keep a list of registered iommus
 * @ops: iommu-ops for talking to this iommu
 * @dev: struct device for sysfs handling
 * @singleton_group: Used internally for drivers that have only one group
 * @max_pasids: number of supported PASIDs
 * @ready: set once iommu_device_register() has completed successfully
 */
struct iommu_device {
        struct list_head list;
        const struct iommu_ops *ops;
        struct fwnode_handle *fwnode;
        struct device *dev;
        struct iommu_group *singleton_group;
        u32 max_pasids;
        bool ready;
};

/**
 * struct iommu_fault_param - per-device IOMMU fault data
 * @lock: protect pending faults list
 * @users: user counter to manage the lifetime of the data
 * @rcu: rcu head for kfree_rcu()
 * @dev: the device that owns this param
 * @queue: IOPF queue
 * @queue_list: index into queue->devices
 * @partial: faults that are part of a Page Request Group for which the last
 *           request hasn't been submitted yet.
 * @faults: holds the pending faults which need response
 */
struct iommu_fault_param {
        struct mutex lock;
        refcount_t users;
        struct rcu_head rcu;

        struct device *dev;
        struct iopf_queue *queue;
        struct list_head queue_list;

        struct list_head partial;
        struct list_head faults;
};

/**
 * struct dev_iommu - Collection of per-device IOMMU data
 *
 * @fault_param: IOMMU detected device fault reporting data
 * @fwspec:         IOMMU fwspec data
 * @iommu_dev:         IOMMU device this device is linked to
 * @priv:         IOMMU Driver private data
 * @max_pasids:  number of PASIDs this device can consume
 * @attach_deferred: the dma domain attachment is deferred
 * @pci_32bit_workaround: Limit DMA allocations to 32-bit IOVAs
 * @require_direct: device requires IOMMU_RESV_DIRECT regions
 * @shadow_on_flush: IOTLB flushes are used to sync shadow tables
 *
 * TODO: migrate other per device data pointers under iommu_dev_data, e.g.
 *        struct iommu_group        *iommu_group;
 */
struct dev_iommu {
        struct mutex lock;
        struct iommu_fault_param __rcu        *fault_param;
        struct iommu_fwspec                *fwspec;
        struct iommu_device                *iommu_dev;
        void                                *priv;
        u32                                max_pasids;
        u32                                attach_deferred:1;
        u32                                pci_32bit_workaround:1;
        u32                                require_direct:1;
        u32                                shadow_on_flush:1;
};

int iommu_device_register(struct iommu_device *iommu,
                          const struct iommu_ops *ops,
                          struct device *hwdev);
void iommu_device_unregister(struct iommu_device *iommu);
int  iommu_device_sysfs_add(struct iommu_device *iommu,
                            struct device *parent,
                            const struct attribute_group **groups,
                            const char *fmt, ...) __printf(4, 5);
void iommu_device_sysfs_remove(struct iommu_device *iommu);
int  iommu_device_link(struct iommu_device   *iommu, struct device *link);
void iommu_device_unlink(struct iommu_device *iommu, struct device *link);
int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain);

static inline struct iommu_device *dev_to_iommu_device(struct device *dev)
{
        return (struct iommu_device *)dev_get_drvdata(dev);
}

/**
 * iommu_get_iommu_dev - Get iommu_device for a device
 * @dev: an end-point device
 *
 * Note that this function must be called from the iommu_ops
 * to retrieve the iommu_device for a device, which the core code
 * guarentees it will not invoke the op without an attached iommu.
 */
static inline struct iommu_device *__iommu_get_iommu_dev(struct device *dev)
{
        return dev->iommu->iommu_dev;
}

#define iommu_get_iommu_dev(dev, type, member) \
        container_of(__iommu_get_iommu_dev(dev), type, member)

static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
{
        *gather = (struct iommu_iotlb_gather) {
                .start        = ULONG_MAX,
                .freelist = IOMMU_PAGES_LIST_INIT(gather->freelist),
        };
}

extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap);
extern bool iommu_group_has_isolated_msi(struct iommu_group *group);
struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev, unsigned int flags);
static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
{
        return iommu_paging_domain_alloc_flags(dev, 0);
}
extern void iommu_domain_free(struct iommu_domain *domain);
extern int iommu_attach_device(struct iommu_domain *domain,
                               struct device *dev);
extern void iommu_detach_device(struct iommu_domain *domain,
                                struct device *dev);
extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev);
extern struct iommu_domain *iommu_get_dma_domain(struct device *dev);
extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
                     phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova,
                phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
int iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
                size_t size);
extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
                          size_t size);
extern size_t iommu_unmap_fast(struct iommu_domain *domain,
                               unsigned long iova, size_t size,
                               struct iommu_iotlb_gather *iotlb_gather);
extern ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
                            struct scatterlist *sg, unsigned int nents,
                            int prot, gfp_t gfp);
extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
extern void iommu_set_fault_handler(struct iommu_domain *domain,
                        iommu_fault_handler_t handler, void *token);

extern void iommu_get_resv_regions(struct device *dev, struct list_head *list);
extern void iommu_put_resv_regions(struct device *dev, struct list_head *list);
extern void iommu_set_default_passthrough(bool cmd_line);
extern void iommu_set_default_translated(bool cmd_line);
extern bool iommu_default_passthrough(void);
extern struct iommu_resv_region *
iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot,
                        enum iommu_resv_type type, gfp_t gfp);
extern int iommu_get_group_resv_regions(struct iommu_group *group,
                                        struct list_head *head);

extern int iommu_attach_group(struct iommu_domain *domain,
                              struct iommu_group *group);
extern void iommu_detach_group(struct iommu_domain *domain,
                               struct iommu_group *group);
extern struct iommu_group *iommu_group_alloc(void);
extern void *iommu_group_get_iommudata(struct iommu_group *group);
extern void iommu_group_set_iommudata(struct iommu_group *group,
                                      void *iommu_data,
                                      void (*release)(void *iommu_data));
extern int iommu_group_set_name(struct iommu_group *group, const char *name);
extern int iommu_group_add_device(struct iommu_group *group,
                                  struct device *dev);
extern void iommu_group_remove_device(struct device *dev);
extern int iommu_group_for_each_dev(struct iommu_group *group, void *data,
                                    int (*fn)(struct device *, void *));
extern struct iommu_group *iommu_group_get(struct device *dev);
extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group);
extern void iommu_group_put(struct iommu_group *group);

extern int iommu_group_id(struct iommu_group *group);
extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *);

int iommu_set_pgtable_quirks(struct iommu_domain *domain,
                unsigned long quirks);

void iommu_set_dma_strict(void);

extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
                              unsigned long iova, int flags);

static inline void iommu_flush_iotlb_all(struct iommu_domain *domain)
{
        if (domain->ops->flush_iotlb_all)
                domain->ops->flush_iotlb_all(domain);
}

static inline void iommu_iotlb_sync(struct iommu_domain *domain,
                                  struct iommu_iotlb_gather *iotlb_gather)
{
        if (domain->ops->iotlb_sync &&
            likely(iotlb_gather->start < iotlb_gather->end))
                domain->ops->iotlb_sync(domain, iotlb_gather);

        iommu_iotlb_gather_init(iotlb_gather);
}

/**
 * iommu_iotlb_gather_is_disjoint - Checks whether a new range is disjoint
 *
 * @gather: TLB gather data
 * @iova: start of page to invalidate
 * @size: size of page to invalidate
 *
 * Helper for IOMMU drivers to check whether a new range and the gathered range
 * are disjoint. For many IOMMUs, flushing the IOMMU in this case is better
 * than merging the two, which might lead to unnecessary invalidations.
 */
static inline
bool iommu_iotlb_gather_is_disjoint(struct iommu_iotlb_gather *gather,
                                    unsigned long iova, size_t size)
{
        unsigned long start = iova, end = start + size - 1;

        return gather->end != 0 &&
                (end + 1 < gather->start || start > gather->end + 1);
}


/**
 * iommu_iotlb_gather_add_range - Gather for address-based TLB invalidation
 * @gather: TLB gather data
 * @iova: start of page to invalidate
 * @size: size of page to invalidate
 *
 * Helper for IOMMU drivers to build arbitrarily-sized invalidation commands
 * where only the address range matters, and simply minimising intermediate
 * syncs is preferred.
 */
static inline void iommu_iotlb_gather_add_range(struct iommu_iotlb_gather *gather,
                                                unsigned long iova, size_t size)
{
        unsigned long end = iova + size - 1;

        if (gather->start > iova)
                gather->start = iova;
        if (gather->end < end)
                gather->end = end;
}

/**
 * iommu_iotlb_gather_add_page - Gather for page-based TLB invalidation
 * @domain: IOMMU domain to be invalidated
 * @gather: TLB gather data
 * @iova: start of page to invalidate
 * @size: size of page to invalidate
 *
 * Helper for IOMMU drivers to build invalidation commands based on individual
 * pages, or with page size/table level hints which cannot be gathered if they
 * differ.
 */
static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
                                               struct iommu_iotlb_gather *gather,
                                               unsigned long iova, size_t size)
{
        /*
         * If the new page is disjoint from the current range or is mapped at
         * a different granularity, then sync the TLB so that the gather
         * structure can be rewritten.
         */
        if ((gather->pgsize && gather->pgsize != size) ||
            iommu_iotlb_gather_is_disjoint(gather, iova, size))
                iommu_iotlb_sync(domain, gather);

        gather->pgsize = size;
        iommu_iotlb_gather_add_range(gather, iova, size);
}

static inline bool iommu_iotlb_gather_queued(struct iommu_iotlb_gather *gather)
{
        return gather && gather->queued;
}

static inline void iommu_dirty_bitmap_init(struct iommu_dirty_bitmap *dirty,
                                           struct iova_bitmap *bitmap,
                                           struct iommu_iotlb_gather *gather)
{
        if (gather)
                iommu_iotlb_gather_init(gather);

        dirty->bitmap = bitmap;
        dirty->gather = gather;
}

static inline void iommu_dirty_bitmap_record(struct iommu_dirty_bitmap *dirty,
                                             unsigned long iova,
                                             unsigned long length)
{
        if (dirty->bitmap)
                iova_bitmap_set(dirty->bitmap, iova, length);

        if (dirty->gather)
                iommu_iotlb_gather_add_range(dirty->gather, iova, length);
}

/* PCI device grouping function */
extern struct iommu_group *pci_device_group(struct device *dev);
/* Generic device grouping function */
extern struct iommu_group *generic_device_group(struct device *dev);
/* FSL-MC device grouping function */
struct iommu_group *fsl_mc_device_group(struct device *dev);
extern struct iommu_group *generic_single_device_group(struct device *dev);

/**
 * struct iommu_fwspec - per-device IOMMU instance data
 * @iommu_fwnode: firmware handle for this device's IOMMU
 * @flags: IOMMU_FWSPEC_* flags
 * @num_ids: number of associated device IDs
 * @ids: IDs which this device may present to the IOMMU
 *
 * Note that the IDs (and any other information, really) stored in this structure should be
 * considered private to the IOMMU device driver and are not to be used directly by IOMMU
 * consumers.
 */
struct iommu_fwspec {
        struct fwnode_handle        *iommu_fwnode;
        u32                        flags;
        unsigned int                num_ids;
        u32                        ids[];
};

/* ATS is supported */
#define IOMMU_FWSPEC_PCI_RC_ATS                        (1 << 0)
/* CANWBS is supported */
#define IOMMU_FWSPEC_PCI_RC_CANWBS                (1 << 1)

/*
 * An iommu attach handle represents a relationship between an iommu domain
 * and a PASID or RID of a device. It is allocated and managed by the component
 * that manages the domain and is stored in the iommu group during the time the
 * domain is attached.
 */
struct iommu_attach_handle {
        struct iommu_domain                *domain;
};

/**
 * struct iommu_sva - handle to a device-mm bond
 */
struct iommu_sva {
        struct iommu_attach_handle        handle;
        struct device                        *dev;
        refcount_t                        users;
};

struct iommu_mm_data {
        u32                        pasid;
        struct mm_struct        *mm;
        struct list_head        sva_domains;
        struct list_head        mm_list_elm;
};

int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode);
int iommu_fwspec_add_ids(struct device *dev, const u32 *ids, int num_ids);

static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
{
        if (dev->iommu)
                return dev->iommu->fwspec;
        else
                return NULL;
}

static inline void dev_iommu_fwspec_set(struct device *dev,
                                        struct iommu_fwspec *fwspec)
{
        dev->iommu->fwspec = fwspec;
}

static inline void *dev_iommu_priv_get(struct device *dev)
{
        if (dev->iommu)
                return dev->iommu->priv;
        else
                return NULL;
}

void dev_iommu_priv_set(struct device *dev, void *priv);

extern struct mutex iommu_probe_device_lock;
int iommu_probe_device(struct device *dev);

int iommu_device_use_default_domain(struct device *dev);
void iommu_device_unuse_default_domain(struct device *dev);

int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner);
void iommu_group_release_dma_owner(struct iommu_group *group);
bool iommu_group_dma_owner_claimed(struct iommu_group *group);

int iommu_device_claim_dma_owner(struct device *dev, void *owner);
void iommu_device_release_dma_owner(struct device *dev);

int iommu_attach_device_pasid(struct iommu_domain *domain,
                              struct device *dev, ioasid_t pasid,
                              struct iommu_attach_handle *handle);
void iommu_detach_device_pasid(struct iommu_domain *domain,
                               struct device *dev, ioasid_t pasid);
ioasid_t iommu_alloc_global_pasid(struct device *dev);
void iommu_free_global_pasid(ioasid_t pasid);

/* PCI device reset functions */
int pci_dev_reset_iommu_prepare(struct pci_dev *pdev);
void pci_dev_reset_iommu_done(struct pci_dev *pdev);
#else /* CONFIG_IOMMU_API */

struct iommu_ops {};
struct iommu_group {};
struct iommu_fwspec {};
struct iommu_device {};
struct iommu_fault_param {};
struct iommu_iotlb_gather {};
struct iommu_dirty_bitmap {};
struct iommu_dirty_ops {};

static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
{
        return false;
}

static inline struct iommu_domain *iommu_paging_domain_alloc_flags(struct device *dev,
                                                     unsigned int flags)
{
        return ERR_PTR(-ENODEV);
}

static inline struct iommu_domain *iommu_paging_domain_alloc(struct device *dev)
{
        return ERR_PTR(-ENODEV);
}

static inline void iommu_domain_free(struct iommu_domain *domain)
{
}

static inline int iommu_attach_device(struct iommu_domain *domain,
                                      struct device *dev)
{
        return -ENODEV;
}

static inline void iommu_detach_device(struct iommu_domain *domain,
                                       struct device *dev)
{
}

static inline struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
{
        return NULL;
}

static inline int iommu_map(struct iommu_domain *domain, unsigned long iova,
                            phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
{
        return -ENODEV;
}

static inline size_t iommu_unmap(struct iommu_domain *domain,
                                 unsigned long iova, size_t size)
{
        return 0;
}

static inline size_t iommu_unmap_fast(struct iommu_domain *domain,
                                      unsigned long iova, int gfp_order,
                                      struct iommu_iotlb_gather *iotlb_gather)
{
        return 0;
}

static inline ssize_t iommu_map_sg(struct iommu_domain *domain,
                                   unsigned long iova, struct scatterlist *sg,
                                   unsigned int nents, int prot, gfp_t gfp)
{
        return -ENODEV;
}

static inline void iommu_flush_iotlb_all(struct iommu_domain *domain)
{
}

static inline void iommu_iotlb_sync(struct iommu_domain *domain,
                                  struct iommu_iotlb_gather *iotlb_gather)
{
}

static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
{
        return 0;
}

static inline void iommu_set_fault_handler(struct iommu_domain *domain,
                                iommu_fault_handler_t handler, void *token)
{
}

static inline void iommu_get_resv_regions(struct device *dev,
                                        struct list_head *list)
{
}

static inline void iommu_put_resv_regions(struct device *dev,
                                        struct list_head *list)
{
}

static inline int iommu_get_group_resv_regions(struct iommu_group *group,
                                               struct list_head *head)
{
        return -ENODEV;
}

static inline void iommu_set_default_passthrough(bool cmd_line)
{
}

static inline void iommu_set_default_translated(bool cmd_line)
{
}

static inline bool iommu_default_passthrough(void)
{
        return true;
}

static inline int iommu_attach_group(struct iommu_domain *domain,
                                     struct iommu_group *group)
{
        return -ENODEV;
}

static inline void iommu_detach_group(struct iommu_domain *domain,
                                      struct iommu_group *group)
{
}

static inline struct iommu_group *iommu_group_alloc(void)
{
        return ERR_PTR(-ENODEV);
}

static inline void *iommu_group_get_iommudata(struct iommu_group *group)
{
        return NULL;
}

static inline void iommu_group_set_iommudata(struct iommu_group *group,
                                             void *iommu_data,
                                             void (*release)(void *iommu_data))
{
}

static inline int iommu_group_set_name(struct iommu_group *group,
                                       const char *name)
{
        return -ENODEV;
}

static inline int iommu_group_add_device(struct iommu_group *group,
                                         struct device *dev)
{
        return -ENODEV;
}

static inline void iommu_group_remove_device(struct device *dev)
{
}

static inline int iommu_group_for_each_dev(struct iommu_group *group,
                                           void *data,
                                           int (*fn)(struct device *, void *))
{
        return -ENODEV;
}

static inline struct iommu_group *iommu_group_get(struct device *dev)
{
        return NULL;
}

static inline void iommu_group_put(struct iommu_group *group)
{
}

static inline int iommu_group_id(struct iommu_group *group)
{
        return -ENODEV;
}

static inline int iommu_set_pgtable_quirks(struct iommu_domain *domain,
                unsigned long quirks)
{
        return 0;
}

static inline int iommu_device_register(struct iommu_device *iommu,
                                        const struct iommu_ops *ops,
                                        struct device *hwdev)
{
        return -ENODEV;
}

static inline struct iommu_device *dev_to_iommu_device(struct device *dev)
{
        return NULL;
}

static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
{
}

static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
                                               struct iommu_iotlb_gather *gather,
                                               unsigned long iova, size_t size)
{
}

static inline bool iommu_iotlb_gather_queued(struct iommu_iotlb_gather *gather)
{
        return false;
}

static inline void iommu_dirty_bitmap_init(struct iommu_dirty_bitmap *dirty,
                                           struct iova_bitmap *bitmap,
                                           struct iommu_iotlb_gather *gather)
{
}

static inline void iommu_dirty_bitmap_record(struct iommu_dirty_bitmap *dirty,
                                             unsigned long iova,
                                             unsigned long length)
{
}

static inline void iommu_device_unregister(struct iommu_device *iommu)
{
}

static inline int  iommu_device_sysfs_add(struct iommu_device *iommu,
                                          struct device *parent,
                                          const struct attribute_group **groups,
                                          const char *fmt, ...)
{
        return -ENODEV;
}

static inline void iommu_device_sysfs_remove(struct iommu_device *iommu)
{
}

static inline int iommu_device_link(struct device *dev, struct device *link)
{
        return -EINVAL;
}

static inline void iommu_device_unlink(struct device *dev, struct device *link)
{
}

static inline int iommu_fwspec_init(struct device *dev,
                                    struct fwnode_handle *iommu_fwnode)
{
        return -ENODEV;
}

static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids,
                                       int num_ids)
{
        return -ENODEV;
}

static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
{
        return NULL;
}

static inline int iommu_device_use_default_domain(struct device *dev)
{
        return 0;
}

static inline void iommu_device_unuse_default_domain(struct device *dev)
{
}

static inline int
iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
{
        return -ENODEV;
}

static inline void iommu_group_release_dma_owner(struct iommu_group *group)
{
}

static inline bool iommu_group_dma_owner_claimed(struct iommu_group *group)
{
        return false;
}

static inline void iommu_device_release_dma_owner(struct device *dev)
{
}

static inline int iommu_device_claim_dma_owner(struct device *dev, void *owner)
{
        return -ENODEV;
}

static inline int iommu_attach_device_pasid(struct iommu_domain *domain,
                                            struct device *dev, ioasid_t pasid,
                                            struct iommu_attach_handle *handle)
{
        return -ENODEV;
}

static inline void iommu_detach_device_pasid(struct iommu_domain *domain,
                                             struct device *dev, ioasid_t pasid)
{
}

static inline ioasid_t iommu_alloc_global_pasid(struct device *dev)
{
        return IOMMU_PASID_INVALID;
}

static inline void iommu_free_global_pasid(ioasid_t pasid) {}

static inline int pci_dev_reset_iommu_prepare(struct pci_dev *pdev)
{
        return 0;
}

static inline void pci_dev_reset_iommu_done(struct pci_dev *pdev)
{
}
#endif /* CONFIG_IOMMU_API */

#ifdef CONFIG_IRQ_MSI_IOMMU
#ifdef CONFIG_IOMMU_API
int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr);
#else
static inline int iommu_dma_prepare_msi(struct msi_desc *desc,
                                        phys_addr_t msi_addr)
{
        return 0;
}
#endif /* CONFIG_IOMMU_API */
#endif /* CONFIG_IRQ_MSI_IOMMU */

#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API)
void iommu_group_mutex_assert(struct device *dev);
#else
static inline void iommu_group_mutex_assert(struct device *dev)
{
}
#endif

/**
 * iommu_map_sgtable - Map the given buffer to the IOMMU domain
 * @domain:        The IOMMU domain to perform the mapping
 * @iova:        The start address to map the buffer
 * @sgt:        The sg_table object describing the buffer
 * @prot:        IOMMU protection bits
 *
 * Creates a mapping at @iova for the buffer described by a scatterlist
 * stored in the given sg_table object in the provided IOMMU domain.
 */
static inline ssize_t iommu_map_sgtable(struct iommu_domain *domain,
                        unsigned long iova, struct sg_table *sgt, int prot)
{
        return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot,
                            GFP_KERNEL);
}

#ifdef CONFIG_IOMMU_DEBUGFS
extern        struct dentry *iommu_debugfs_dir;
void iommu_debugfs_setup(void);
#else
static inline void iommu_debugfs_setup(void) {}
#endif

#ifdef CONFIG_IOMMU_DMA
int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base);
#else /* CONFIG_IOMMU_DMA */
static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
{
        return -ENODEV;
}
#endif        /* CONFIG_IOMMU_DMA */

/*
 * Newer generations of Tegra SoCs require devices' stream IDs to be directly programmed into
 * some registers. These are always paired with a Tegra SMMU or ARM SMMU, for which the contents
 * of the struct iommu_fwspec are known. Use this helper to formalize access to these internals.
 */
#define TEGRA_STREAM_ID_BYPASS 0x7f

static inline bool tegra_dev_iommu_get_stream_id(struct device *dev, u32 *stream_id)
{
#ifdef CONFIG_IOMMU_API
        struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);

        if (fwspec && fwspec->num_ids == 1) {
                *stream_id = fwspec->ids[0] & 0xffff;
                return true;
        }
#endif

        return false;
}

#ifdef CONFIG_IOMMU_MM_DATA
static inline void mm_pasid_init(struct mm_struct *mm)
{
        /*
         * During dup_mm(), a new mm will be memcpy'd from an old one and that makes
         * the new mm and the old one point to a same iommu_mm instance. When either
         * one of the two mms gets released, the iommu_mm instance is freed, leaving
         * the other mm running into a use-after-free/double-free problem. To avoid
         * the problem, zeroing the iommu_mm pointer of a new mm is needed here.
         */
        mm->iommu_mm = NULL;
}

static inline bool mm_valid_pasid(struct mm_struct *mm)
{
        return READ_ONCE(mm->iommu_mm);
}

static inline u32 mm_get_enqcmd_pasid(struct mm_struct *mm)
{
        struct iommu_mm_data *iommu_mm = READ_ONCE(mm->iommu_mm);

        if (!iommu_mm)
                return IOMMU_PASID_INVALID;
        return iommu_mm->pasid;
}

void mm_pasid_drop(struct mm_struct *mm);
struct iommu_sva *iommu_sva_bind_device(struct device *dev,
                                        struct mm_struct *mm);
void iommu_sva_unbind_device(struct iommu_sva *handle);
u32 iommu_sva_get_pasid(struct iommu_sva *handle);
void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end);
#else
static inline struct iommu_sva *
iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
{
        return ERR_PTR(-ENODEV);
}

static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
{
}

static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
{
        return IOMMU_PASID_INVALID;
}
static inline void mm_pasid_init(struct mm_struct *mm) {}
static inline bool mm_valid_pasid(struct mm_struct *mm) { return false; }

static inline u32 mm_get_enqcmd_pasid(struct mm_struct *mm)
{
        return IOMMU_PASID_INVALID;
}

static inline void mm_pasid_drop(struct mm_struct *mm) {}
static inline void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end) {}
#endif /* CONFIG_IOMMU_SVA */

#ifdef CONFIG_IOMMU_IOPF
int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev);
void iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev);
int iopf_queue_flush_dev(struct device *dev);
struct iopf_queue *iopf_queue_alloc(const char *name);
void iopf_queue_free(struct iopf_queue *queue);
int iopf_queue_discard_partial(struct iopf_queue *queue);
void iopf_free_group(struct iopf_group *group);
int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt);
void iopf_group_response(struct iopf_group *group,
                         enum iommu_page_response_code status);
#else
static inline int
iopf_queue_add_device(struct iopf_queue *queue, struct device *dev)
{
        return -ENODEV;
}

static inline void
iopf_queue_remove_device(struct iopf_queue *queue, struct device *dev)
{
}

static inline int iopf_queue_flush_dev(struct device *dev)
{
        return -ENODEV;
}

static inline struct iopf_queue *iopf_queue_alloc(const char *name)
{
        return NULL;
}

static inline void iopf_queue_free(struct iopf_queue *queue)
{
}

static inline int iopf_queue_discard_partial(struct iopf_queue *queue)
{
        return -ENODEV;
}

static inline void iopf_free_group(struct iopf_group *group)
{
}

static inline int
iommu_report_device_fault(struct device *dev, struct iopf_fault *evt)
{
        return -ENODEV;
}

static inline void iopf_group_response(struct iopf_group *group,
                                       enum iommu_page_response_code status)
{
}
#endif /* CONFIG_IOMMU_IOPF */
#endif /* __LINUX_IOMMU_H */











































































































































































































    5 














    5 




    5 










   22 

















  128 
  126 




  121 
  121 










    6 





   88 
   88 













   88 











   85 














  103 
  103 




















   91 
  100 






   97 


   97 


   97 




   87 
   96 



   96 








   88 




    7 
































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
 */

#include <net/genetlink.h>
#define CREATE_TRACE_POINTS
#include <trace/events/devlink.h>

#include "devl_internal.h"

EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);

DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);

static struct devlink *devlinks_xa_get(unsigned long index)
{
        struct devlink *devlink;

        rcu_read_lock();
        devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED);
        if (!devlink || !devlink_try_get(devlink))
                devlink = NULL;
        rcu_read_unlock();
        return devlink;
}

/* devlink_rels xarray contains 1:1 relationships between
 * devlink object and related nested devlink instance.
 * The xarray index is used to get the nested object from
 * the nested-in object code.
 */
static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1);

#define DEVLINK_REL_IN_USE XA_MARK_0

struct devlink_rel {
        u32 index;
        refcount_t refcount;
        u32 devlink_index;
        struct {
                u32 devlink_index;
                u32 obj_index;
                devlink_rel_notify_cb_t *notify_cb;
                devlink_rel_cleanup_cb_t *cleanup_cb;
                struct delayed_work notify_work;
        } nested_in;
};

static void devlink_rel_free(struct devlink_rel *rel)
{
        xa_erase(&devlink_rels, rel->index);
        kfree(rel);
}

static void __devlink_rel_get(struct devlink_rel *rel)
{
        refcount_inc(&rel->refcount);
}

static void __devlink_rel_put(struct devlink_rel *rel)
{
        if (refcount_dec_and_test(&rel->refcount))
                devlink_rel_free(rel);
}

static void devlink_rel_nested_in_notify_work(struct work_struct *work)
{
        struct devlink_rel *rel = container_of(work, struct devlink_rel,
                                               nested_in.notify_work.work);
        struct devlink *devlink;

        devlink = devlinks_xa_get(rel->nested_in.devlink_index);
        if (!devlink)
                goto rel_put;
        if (!devl_trylock(devlink)) {
                devlink_put(devlink);
                goto reschedule_work;
        }
        if (!devl_is_registered(devlink)) {
                devl_unlock(devlink);
                devlink_put(devlink);
                goto rel_put;
        }
        if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE))
                rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index);
        rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index);
        devl_unlock(devlink);
        devlink_put(devlink);

rel_put:
        __devlink_rel_put(rel);
        return;

reschedule_work:
        schedule_delayed_work(&rel->nested_in.notify_work, 1);
}

static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel)
{
        __devlink_rel_get(rel);
        schedule_delayed_work(&rel->nested_in.notify_work, 0);
}

static struct devlink_rel *devlink_rel_alloc(void)
{
        struct devlink_rel *rel;
        static u32 next;
        int err;

        rel = kzalloc_obj(*rel);
        if (!rel)
                return ERR_PTR(-ENOMEM);

        err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel,
                              xa_limit_32b, &next, GFP_KERNEL);
        if (err < 0) {
                kfree(rel);
                return ERR_PTR(err);
        }

        refcount_set(&rel->refcount, 1);
        INIT_DELAYED_WORK(&rel->nested_in.notify_work,
                          &devlink_rel_nested_in_notify_work);
        return rel;
}

static void devlink_rel_put(struct devlink *devlink)
{
        struct devlink_rel *rel = devlink->rel;

        if (!rel)
                return;
        xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
        devlink_rel_nested_in_notify_work_schedule(rel);
        __devlink_rel_put(rel);
        devlink->rel = NULL;
}

void devlink_rel_nested_in_clear(u32 rel_index)
{
        xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE);
}

int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index,
                              u32 obj_index, devlink_rel_notify_cb_t *notify_cb,
                              devlink_rel_cleanup_cb_t *cleanup_cb,
                              struct devlink *devlink)
{
        struct devlink_rel *rel = devlink_rel_alloc();

        ASSERT_DEVLINK_NOT_REGISTERED(devlink);

        if (IS_ERR(rel))
                return PTR_ERR(rel);

        rel->devlink_index = devlink->index;
        rel->nested_in.devlink_index = devlink_index;
        rel->nested_in.obj_index = obj_index;
        rel->nested_in.notify_cb = notify_cb;
        rel->nested_in.cleanup_cb = cleanup_cb;
        *rel_index = rel->index;
        xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
        devlink->rel = rel;
        return 0;
}

/**
 * devlink_rel_nested_in_notify - Notify the object this devlink
 *                                  instance is nested in.
 * @devlink: devlink
 *
 * This is called upon network namespace change of devlink instance.
 * In case this devlink instance is nested in another devlink object,
 * a notification of a change of this object should be sent
 * over netlink. The parent devlink instance lock needs to be
 * taken during the notification preparation.
 * Since the parent may or may not be locked, 'work' is utilized.
 */
void devlink_rel_nested_in_notify(struct devlink *devlink)
{
        struct devlink_rel *rel = devlink->rel;

        if (!rel)
                return;
        devlink_rel_nested_in_notify_work_schedule(rel);
}

static struct devlink_rel *devlink_rel_find(unsigned long rel_index)
{
        return xa_find(&devlink_rels, &rel_index, rel_index,
                       DEVLINK_REL_IN_USE);
}

static struct devlink *devlink_rel_devlink_get(u32 rel_index)
{
        struct devlink_rel *rel;
        u32 devlink_index;

        if (!rel_index)
                return NULL;
        xa_lock(&devlink_rels);
        rel = devlink_rel_find(rel_index);
        if (rel)
                devlink_index = rel->devlink_index;
        xa_unlock(&devlink_rels);
        if (!rel)
                return NULL;
        return devlinks_xa_get(devlink_index);
}

int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink,
                                   u32 rel_index, int attrtype,
                                   bool *msg_updated)
{
        struct net *net = devlink_net(devlink);
        struct devlink *rel_devlink;
        int err;

        rel_devlink = devlink_rel_devlink_get(rel_index);
        if (!rel_devlink)
                return 0;
        err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype);
        devlink_put(rel_devlink);
        if (!err && msg_updated)
                *msg_updated = true;
        return err;
}

void *devlink_priv(struct devlink *devlink)
{
        return &devlink->priv;
}
EXPORT_SYMBOL_GPL(devlink_priv);

struct devlink *priv_to_devlink(void *priv)
{
        return container_of(priv, struct devlink, priv);
}
EXPORT_SYMBOL_GPL(priv_to_devlink);

struct device *devlink_to_dev(const struct devlink *devlink)
{
        return devlink->dev;
}
EXPORT_SYMBOL_GPL(devlink_to_dev);

const char *devlink_bus_name(const struct devlink *devlink)
{
        return devlink->dev ? devlink->dev->bus->name : DEVLINK_INDEX_BUS_NAME;
}
EXPORT_SYMBOL_GPL(devlink_bus_name);

const char *devlink_dev_name(const struct devlink *devlink)
{
        return devlink->dev ? dev_name(devlink->dev) : devlink->dev_name_index;
}
EXPORT_SYMBOL_GPL(devlink_dev_name);

const char *devlink_dev_driver_name(const struct devlink *devlink)
{
        return devlink->dev_driver->name;
}
EXPORT_SYMBOL_GPL(devlink_dev_driver_name);

struct net *devlink_net(const struct devlink *devlink)
{
        return read_pnet(&devlink->_net);
}
EXPORT_SYMBOL_GPL(devlink_net);

void devl_assert_locked(struct devlink *devlink)
{
        lockdep_assert_held(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_assert_locked);

#ifdef CONFIG_LOCKDEP
/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
bool devl_lock_is_held(struct devlink *devlink)
{
        return lockdep_is_held(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_lock_is_held);
#endif

void devl_lock(struct devlink *devlink)
{
        mutex_lock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_lock);

int devl_trylock(struct devlink *devlink)
{
        return mutex_trylock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_trylock);

void devl_unlock(struct devlink *devlink)
{
        mutex_unlock(&devlink->lock);
}
EXPORT_SYMBOL_GPL(devl_unlock);

/**
 * devlink_try_get() - try to obtain a reference on a devlink instance
 * @devlink: instance to reference
 *
 * Obtain a reference on a devlink instance. A reference on a devlink instance
 * only implies that it's safe to take the instance lock. It does not imply
 * that the instance is registered, use devl_is_registered() after taking
 * the instance lock to check registration status.
 */
struct devlink *__must_check devlink_try_get(struct devlink *devlink)
{
        if (refcount_inc_not_zero(&devlink->refcount))
                return devlink;
        return NULL;
}

static void devlink_release(struct work_struct *work)
{
        struct devlink *devlink;

        devlink = container_of(to_rcu_work(work), struct devlink, rwork);

        mutex_destroy(&devlink->lock);
        lockdep_unregister_key(&devlink->lock_key);
        if (devlink->dev)
                put_device(devlink->dev);
        else
                kfree(devlink->dev_name_index);
        kvfree(devlink);
}

void devlink_put(struct devlink *devlink)
{
        if (refcount_dec_and_test(&devlink->refcount))
                queue_rcu_work(system_percpu_wq, &devlink->rwork);
}

static struct devlink *__devlinks_xa_find_get(struct net *net,
                                              unsigned long *indexp,
                                              unsigned long end)
{
        struct devlink *devlink = NULL;

        rcu_read_lock();
retry:
        devlink = xa_find(&devlinks, indexp, end, DEVLINK_REGISTERED);
        if (!devlink)
                goto unlock;

        if (!devlink_try_get(devlink))
                goto next;
        if (!net_eq(devlink_net(devlink), net)) {
                devlink_put(devlink);
                goto next;
        }
unlock:
        rcu_read_unlock();
        return devlink;

next:
        (*indexp)++;
        goto retry;
}

struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp)
{
        return __devlinks_xa_find_get(net, indexp, ULONG_MAX);
}

struct devlink *devlinks_xa_lookup_get(struct net *net, unsigned long index)
{
        return __devlinks_xa_find_get(net, &index, index);
}

/**
 * devl_register - Register devlink instance
 * @devlink: devlink
 */
int devl_register(struct devlink *devlink)
{
        ASSERT_DEVLINK_NOT_REGISTERED(devlink);
        devl_assert_locked(devlink);

        xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
        devlink_notify_register(devlink);
        devlink_rel_nested_in_notify(devlink);

        return 0;
}
EXPORT_SYMBOL_GPL(devl_register);

void devlink_register(struct devlink *devlink)
{
        devl_lock(devlink);
        devl_register(devlink);
        devl_unlock(devlink);
}
EXPORT_SYMBOL_GPL(devlink_register);

/**
 * devl_unregister - Unregister devlink instance
 * @devlink: devlink
 */
void devl_unregister(struct devlink *devlink)
{
        ASSERT_DEVLINK_REGISTERED(devlink);
        devl_assert_locked(devlink);

        devlink_notify_unregister(devlink);
        xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
        devlink_rel_put(devlink);
}
EXPORT_SYMBOL_GPL(devl_unregister);

void devlink_unregister(struct devlink *devlink)
{
        devl_lock(devlink);
        devl_unregister(devlink);
        devl_unlock(devlink);
}
EXPORT_SYMBOL_GPL(devlink_unregister);

struct devlink *__devlink_alloc(const struct devlink_ops *ops, size_t priv_size,
                                struct net *net, struct device *dev,
                                const struct device_driver *dev_driver)
{
        struct devlink *devlink;
        static u32 last_id;
        int ret;

        WARN_ON(!ops || !dev_driver);
        if (!devlink_reload_actions_valid(ops))
                return NULL;

        devlink = kvzalloc_flex(*devlink, priv, priv_size);
        if (!devlink)
                return NULL;

        ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
                              &last_id, GFP_KERNEL);
        if (ret < 0)
                goto err_xa_alloc;

        if (dev) {
                devlink->dev = get_device(dev);
        } else {
                devlink->dev_name_index = kasprintf(GFP_KERNEL, "%u", devlink->index);
                if (!devlink->dev_name_index)
                        goto err_kasprintf;
        }

        devlink->ops = ops;
        devlink->dev_driver = dev_driver;
        xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC);
        xa_init_flags(&devlink->params, XA_FLAGS_ALLOC);
        xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
        xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC);
        write_pnet(&devlink->_net, net);
        INIT_LIST_HEAD(&devlink->rate_list);
        INIT_LIST_HEAD(&devlink->linecard_list);
        INIT_LIST_HEAD(&devlink->sb_list);
        INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
        INIT_LIST_HEAD(&devlink->resource_list);
        INIT_LIST_HEAD(&devlink->region_list);
        INIT_LIST_HEAD(&devlink->reporter_list);
        INIT_LIST_HEAD(&devlink->trap_list);
        INIT_LIST_HEAD(&devlink->trap_group_list);
        INIT_LIST_HEAD(&devlink->trap_policer_list);
        INIT_RCU_WORK(&devlink->rwork, devlink_release);
        lockdep_register_key(&devlink->lock_key);
        mutex_init(&devlink->lock);
        lockdep_set_class(&devlink->lock, &devlink->lock_key);
        refcount_set(&devlink->refcount, 1);

        return devlink;

err_kasprintf:
        xa_erase(&devlinks, devlink->index);
err_xa_alloc:
        kvfree(devlink);
        return NULL;
}

/**
 *        devlink_alloc_ns - Allocate new devlink instance resources
 *        in specific namespace
 *
 *        @ops: ops
 *        @priv_size: size of user private data
 *        @net: net namespace
 *        @dev: parent device
 *
 *        Allocate new devlink instance resources, including devlink index
 *        and name.
 */
struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
                                 size_t priv_size, struct net *net,
                                 struct device *dev)
{
        WARN_ON(!dev);
        return __devlink_alloc(ops, priv_size, net, dev, dev->driver);
}
EXPORT_SYMBOL_GPL(devlink_alloc_ns);

/**
 *        devlink_free - Free devlink instance resources
 *
 *        @devlink: devlink
 */
void devlink_free(struct devlink *devlink)
{
        ASSERT_DEVLINK_NOT_REGISTERED(devlink);

        devlink_rel_put(devlink);

        WARN_ON(!list_empty(&devlink->trap_policer_list));
        WARN_ON(!list_empty(&devlink->trap_group_list));
        WARN_ON(!list_empty(&devlink->trap_list));
        WARN_ON(!list_empty(&devlink->reporter_list));
        WARN_ON(!list_empty(&devlink->region_list));
        WARN_ON(!list_empty(&devlink->resource_list));
        WARN_ON(!list_empty(&devlink->dpipe_table_list));
        WARN_ON(!list_empty(&devlink->sb_list));
        WARN_ON(devlink_rates_check(devlink, NULL, NULL));
        WARN_ON(!list_empty(&devlink->linecard_list));
        WARN_ON(!xa_empty(&devlink->ports));

        xa_destroy(&devlink->nested_rels);
        xa_destroy(&devlink->snapshot_ids);
        xa_destroy(&devlink->params);
        xa_destroy(&devlink->ports);

        xa_erase(&devlinks, devlink->index);

        devlink_put(devlink);
}
EXPORT_SYMBOL_GPL(devlink_free);

static void __net_exit devlink_pernet_pre_exit(struct net *net)
{
        struct devlink *devlink;
        u32 actions_performed;
        unsigned long index;
        int err;

        /* In case network namespace is getting destroyed, reload
         * all devlink instances from this namespace into init_net.
         */
        devlinks_xa_for_each_registered_get(net, index, devlink) {
                devl_dev_lock(devlink, true);
                err = 0;
                if (devl_is_registered(devlink))
                        err = devlink_reload(devlink, &init_net,
                                             DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
                                             DEVLINK_RELOAD_LIMIT_UNSPEC,
                                             &actions_performed, NULL);
                devl_dev_unlock(devlink, true);
                devlink_put(devlink);
                if (err && err != -EOPNOTSUPP)
                        pr_warn("Failed to reload devlink instance into init_net\n");
        }
}

static struct pernet_operations devlink_pernet_ops __net_initdata = {
        .pre_exit = devlink_pernet_pre_exit,
};

static struct notifier_block devlink_port_netdevice_nb = {
        .notifier_call = devlink_port_netdevice_event,
};

static int __init devlink_init(void)
{
        int err;

        err = register_pernet_subsys(&devlink_pernet_ops);
        if (err)
                goto out;
        err = genl_register_family(&devlink_nl_family);
        if (err)
                goto out_unreg_pernet_subsys;
        err = register_netdevice_notifier(&devlink_port_netdevice_nb);
        if (!err)
                return 0;

        genl_unregister_family(&devlink_nl_family);

out_unreg_pernet_subsys:
        unregister_pernet_subsys(&devlink_pernet_ops);
out:
        WARN_ON(err);
        return err;
}

subsys_initcall(devlink_init);




















































































































































































































































































































































































































































































































































































































































































































































































































































   19 










   19 


   19 











   19 



























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
/*
 * Copyright (c) 2006 Intel Corporation.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/completion.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/bitops.h>
#include <linux/random.h>

#include <rdma/ib_cache.h>
#include "sa.h"

static int mcast_add_one(struct ib_device *device);
static void mcast_remove_one(struct ib_device *device, void *client_data);

static struct ib_client mcast_client = {
        .name   = "ib_multicast",
        .add    = mcast_add_one,
        .remove = mcast_remove_one
};

static struct ib_sa_client        sa_client;
static struct workqueue_struct        *mcast_wq;
static union ib_gid mgid0;

struct mcast_device;

struct mcast_port {
        struct mcast_device        *dev;
        spinlock_t                lock;
        struct rb_root                table;
        refcount_t                refcount;
        struct completion        comp;
        u32                        port_num;
};

struct mcast_device {
        struct ib_device        *device;
        struct ib_event_handler        event_handler;
        int                        start_port;
        int                        end_port;
        struct mcast_port        port[];
};

enum mcast_state {
        MCAST_JOINING,
        MCAST_MEMBER,
        MCAST_ERROR,
};

enum mcast_group_state {
        MCAST_IDLE,
        MCAST_BUSY,
        MCAST_GROUP_ERROR,
        MCAST_PKEY_EVENT
};

enum {
        MCAST_INVALID_PKEY_INDEX = 0xFFFF
};

struct mcast_member;

struct mcast_group {
        struct ib_sa_mcmember_rec rec;
        struct rb_node                node;
        struct mcast_port        *port;
        spinlock_t                lock;
        struct work_struct        work;
        struct list_head        pending_list;
        struct list_head        active_list;
        struct mcast_member        *last_join;
        int                        members[NUM_JOIN_MEMBERSHIP_TYPES];
        atomic_t                refcount;
        enum mcast_group_state        state;
        struct ib_sa_query        *query;
        u16                        pkey_index;
        u8                        leave_state;
        int                        retries;
};

struct mcast_member {
        struct ib_sa_multicast        multicast;
        struct ib_sa_client        *client;
        struct mcast_group        *group;
        struct list_head        list;
        enum mcast_state        state;
        refcount_t                refcount;
        struct completion        comp;
};

static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
                         void *context);
static void leave_handler(int status, struct ib_sa_mcmember_rec *rec,
                          void *context);

static struct mcast_group *mcast_find(struct mcast_port *port,
                                      union ib_gid *mgid)
{
        struct rb_node *node = port->table.rb_node;
        struct mcast_group *group;
        int ret;

        while (node) {
                group = rb_entry(node, struct mcast_group, node);
                ret = memcmp(mgid->raw, group->rec.mgid.raw, sizeof *mgid);
                if (!ret)
                        return group;

                if (ret < 0)
                        node = node->rb_left;
                else
                        node = node->rb_right;
        }
        return NULL;
}

static struct mcast_group *mcast_insert(struct mcast_port *port,
                                        struct mcast_group *group,
                                        int allow_duplicates)
{
        struct rb_node **link = &port->table.rb_node;
        struct rb_node *parent = NULL;
        struct mcast_group *cur_group;
        int ret;

        while (*link) {
                parent = *link;
                cur_group = rb_entry(parent, struct mcast_group, node);

                ret = memcmp(group->rec.mgid.raw, cur_group->rec.mgid.raw,
                             sizeof group->rec.mgid);
                if (ret < 0)
                        link = &(*link)->rb_left;
                else if (ret > 0)
                        link = &(*link)->rb_right;
                else if (allow_duplicates)
                        link = &(*link)->rb_left;
                else
                        return cur_group;
        }
        rb_link_node(&group->node, parent, link);
        rb_insert_color(&group->node, &port->table);
        return NULL;
}

static void deref_port(struct mcast_port *port)
{
        if (refcount_dec_and_test(&port->refcount))
                complete(&port->comp);
}

static void release_group(struct mcast_group *group)
{
        struct mcast_port *port = group->port;
        unsigned long flags;

        spin_lock_irqsave(&port->lock, flags);
        if (atomic_dec_and_test(&group->refcount)) {
                rb_erase(&group->node, &port->table);
                spin_unlock_irqrestore(&port->lock, flags);
                kfree(group);
                deref_port(port);
        } else
                spin_unlock_irqrestore(&port->lock, flags);
}

static void deref_member(struct mcast_member *member)
{
        if (refcount_dec_and_test(&member->refcount))
                complete(&member->comp);
}

static void queue_join(struct mcast_member *member)
{
        struct mcast_group *group = member->group;
        unsigned long flags;

        spin_lock_irqsave(&group->lock, flags);
        list_add_tail(&member->list, &group->pending_list);
        if (group->state == MCAST_IDLE) {
                group->state = MCAST_BUSY;
                atomic_inc(&group->refcount);
                queue_work(mcast_wq, &group->work);
        }
        spin_unlock_irqrestore(&group->lock, flags);
}

/*
 * A multicast group has four types of members: full member, non member,
 * sendonly non member and sendonly full member.
 * We need to keep track of the number of members of each
 * type based on their join state.  Adjust the number of members the belong to
 * the specified join states.
 */
static void adjust_membership(struct mcast_group *group, u8 join_state, int inc)
{
        int i;

        for (i = 0; i < NUM_JOIN_MEMBERSHIP_TYPES; i++, join_state >>= 1)
                if (join_state & 0x1)
                        group->members[i] += inc;
}

/*
 * If a multicast group has zero members left for a particular join state, but
 * the group is still a member with the SA, we need to leave that join state.
 * Determine which join states we still belong to, but that do not have any
 * active members.
 */
static u8 get_leave_state(struct mcast_group *group)
{
        u8 leave_state = 0;
        int i;

        for (i = 0; i < NUM_JOIN_MEMBERSHIP_TYPES; i++)
                if (!group->members[i])
                        leave_state |= (0x1 << i);

        return leave_state & group->rec.join_state;
}

static int check_selector(ib_sa_comp_mask comp_mask,
                          ib_sa_comp_mask selector_mask,
                          ib_sa_comp_mask value_mask,
                          u8 selector, u8 src_value, u8 dst_value)
{
        int err;

        if (!(comp_mask & selector_mask) || !(comp_mask & value_mask))
                return 0;

        switch (selector) {
        case IB_SA_GT:
                err = (src_value <= dst_value);
                break;
        case IB_SA_LT:
                err = (src_value >= dst_value);
                break;
        case IB_SA_EQ:
                err = (src_value != dst_value);
                break;
        default:
                err = 0;
                break;
        }

        return err;
}

static int cmp_rec(struct ib_sa_mcmember_rec *src,
                   struct ib_sa_mcmember_rec *dst, ib_sa_comp_mask comp_mask)
{
        /* MGID must already match */

        if (comp_mask & IB_SA_MCMEMBER_REC_PORT_GID &&
            memcmp(&src->port_gid, &dst->port_gid, sizeof src->port_gid))
                return -EINVAL;
        if (comp_mask & IB_SA_MCMEMBER_REC_QKEY && src->qkey != dst->qkey)
                return -EINVAL;
        if (comp_mask & IB_SA_MCMEMBER_REC_MLID && src->mlid != dst->mlid)
                return -EINVAL;
        if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_MTU_SELECTOR,
                           IB_SA_MCMEMBER_REC_MTU, dst->mtu_selector,
                           src->mtu, dst->mtu))
                return -EINVAL;
        if (comp_mask & IB_SA_MCMEMBER_REC_TRAFFIC_CLASS &&
            src->traffic_class != dst->traffic_class)
                return -EINVAL;
        if (comp_mask & IB_SA_MCMEMBER_REC_PKEY && src->pkey != dst->pkey)
                return -EINVAL;
        if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_RATE_SELECTOR,
                           IB_SA_MCMEMBER_REC_RATE, dst->rate_selector,
                           src->rate, dst->rate))
                return -EINVAL;
        if (check_selector(comp_mask,
                           IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR,
                           IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME,
                           dst->packet_life_time_selector,
                           src->packet_life_time, dst->packet_life_time))
                return -EINVAL;
        if (comp_mask & IB_SA_MCMEMBER_REC_SL && src->sl != dst->sl)
                return -EINVAL;
        if (comp_mask & IB_SA_MCMEMBER_REC_FLOW_LABEL &&
            src->flow_label != dst->flow_label)
                return -EINVAL;
        if (comp_mask & IB_SA_MCMEMBER_REC_HOP_LIMIT &&
            src->hop_limit != dst->hop_limit)
                return -EINVAL;
        if (comp_mask & IB_SA_MCMEMBER_REC_SCOPE && src->scope != dst->scope)
                return -EINVAL;

        /* join_state checked separately, proxy_join ignored */

        return 0;
}

static int send_join(struct mcast_group *group, struct mcast_member *member)
{
        struct mcast_port *port = group->port;
        int ret;

        group->last_join = member;
        ret = ib_sa_mcmember_rec_query(&sa_client, port->dev->device,
                                       port->port_num, IB_MGMT_METHOD_SET,
                                       &member->multicast.rec,
                                       member->multicast.comp_mask,
                                       3000, GFP_KERNEL, join_handler, group,
                                       &group->query);
        return (ret > 0) ? 0 : ret;
}

static int send_leave(struct mcast_group *group, u8 leave_state)
{
        struct mcast_port *port = group->port;
        struct ib_sa_mcmember_rec rec;
        int ret;

        rec = group->rec;
        rec.join_state = leave_state;
        group->leave_state = leave_state;

        ret = ib_sa_mcmember_rec_query(&sa_client, port->dev->device,
                                       port->port_num, IB_SA_METHOD_DELETE, &rec,
                                       IB_SA_MCMEMBER_REC_MGID     |
                                       IB_SA_MCMEMBER_REC_PORT_GID |
                                       IB_SA_MCMEMBER_REC_JOIN_STATE,
                                       3000, GFP_KERNEL, leave_handler,
                                       group, &group->query);
        return (ret > 0) ? 0 : ret;
}

static void join_group(struct mcast_group *group, struct mcast_member *member,
                       u8 join_state)
{
        member->state = MCAST_MEMBER;
        adjust_membership(group, join_state, 1);
        group->rec.join_state |= join_state;
        member->multicast.rec = group->rec;
        member->multicast.rec.join_state = join_state;
        list_move(&member->list, &group->active_list);
}

static int fail_join(struct mcast_group *group, struct mcast_member *member,
                     int status)
{
        spin_lock_irq(&group->lock);
        list_del_init(&member->list);
        spin_unlock_irq(&group->lock);
        return member->multicast.callback(status, &member->multicast);
}

static void process_group_error(struct mcast_group *group)
{
        struct mcast_member *member;
        int ret = 0;
        u16 pkey_index;

        if (group->state == MCAST_PKEY_EVENT)
                ret = ib_find_pkey(group->port->dev->device,
                                   group->port->port_num,
                                   be16_to_cpu(group->rec.pkey), &pkey_index);

        spin_lock_irq(&group->lock);
        if (group->state == MCAST_PKEY_EVENT && !ret &&
            group->pkey_index == pkey_index)
                goto out;

        while (!list_empty(&group->active_list)) {
                member = list_entry(group->active_list.next,
                                    struct mcast_member, list);
                refcount_inc(&member->refcount);
                list_del_init(&member->list);
                adjust_membership(group, member->multicast.rec.join_state, -1);
                member->state = MCAST_ERROR;
                spin_unlock_irq(&group->lock);

                ret = member->multicast.callback(-ENETRESET,
                                                 &member->multicast);
                deref_member(member);
                if (ret)
                        ib_sa_free_multicast(&member->multicast);
                spin_lock_irq(&group->lock);
        }

        group->rec.join_state = 0;
out:
        group->state = MCAST_BUSY;
        spin_unlock_irq(&group->lock);
}

static void mcast_work_handler(struct work_struct *work)
{
        struct mcast_group *group;
        struct mcast_member *member;
        struct ib_sa_multicast *multicast;
        int status, ret;
        u8 join_state;

        group = container_of(work, typeof(*group), work);
retest:
        spin_lock_irq(&group->lock);
        while (!list_empty(&group->pending_list) ||
               (group->state != MCAST_BUSY)) {

                if (group->state != MCAST_BUSY) {
                        spin_unlock_irq(&group->lock);
                        process_group_error(group);
                        goto retest;
                }

                member = list_entry(group->pending_list.next,
                                    struct mcast_member, list);
                multicast = &member->multicast;
                join_state = multicast->rec.join_state;
                refcount_inc(&member->refcount);

                if (join_state == (group->rec.join_state & join_state)) {
                        status = cmp_rec(&group->rec, &multicast->rec,
                                         multicast->comp_mask);
                        if (!status)
                                join_group(group, member, join_state);
                        else
                                list_del_init(&member->list);
                        spin_unlock_irq(&group->lock);
                        ret = multicast->callback(status, multicast);
                } else {
                        spin_unlock_irq(&group->lock);
                        status = send_join(group, member);
                        if (!status) {
                                deref_member(member);
                                return;
                        }
                        ret = fail_join(group, member, status);
                }

                deref_member(member);
                if (ret)
                        ib_sa_free_multicast(&member->multicast);
                spin_lock_irq(&group->lock);
        }

        join_state = get_leave_state(group);
        if (join_state) {
                group->rec.join_state &= ~join_state;
                spin_unlock_irq(&group->lock);
                if (send_leave(group, join_state))
                        goto retest;
        } else {
                group->state = MCAST_IDLE;
                spin_unlock_irq(&group->lock);
                release_group(group);
        }
}

/*
 * Fail a join request if it is still active - at the head of the pending queue.
 */
static void process_join_error(struct mcast_group *group, int status)
{
        struct mcast_member *member;
        int ret;

        spin_lock_irq(&group->lock);
        member = list_entry(group->pending_list.next,
                            struct mcast_member, list);
        if (group->last_join == member) {
                refcount_inc(&member->refcount);
                list_del_init(&member->list);
                spin_unlock_irq(&group->lock);
                ret = member->multicast.callback(status, &member->multicast);
                deref_member(member);
                if (ret)
                        ib_sa_free_multicast(&member->multicast);
        } else
                spin_unlock_irq(&group->lock);
}

static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
                         void *context)
{
        struct mcast_group *group = context;
        u16 pkey_index = MCAST_INVALID_PKEY_INDEX;

        if (status)
                process_join_error(group, status);
        else {
                int mgids_changed, is_mgid0;

                if (ib_find_pkey(group->port->dev->device,
                                 group->port->port_num, be16_to_cpu(rec->pkey),
                                 &pkey_index))
                        pkey_index = MCAST_INVALID_PKEY_INDEX;

                spin_lock_irq(&group->port->lock);
                if (group->state == MCAST_BUSY &&
                    group->pkey_index == MCAST_INVALID_PKEY_INDEX)
                        group->pkey_index = pkey_index;
                mgids_changed = memcmp(&rec->mgid, &group->rec.mgid,
                                       sizeof(group->rec.mgid));
                group->rec = *rec;
                if (mgids_changed) {
                        rb_erase(&group->node, &group->port->table);
                        is_mgid0 = !memcmp(&mgid0, &group->rec.mgid,
                                           sizeof(mgid0));
                        mcast_insert(group->port, group, is_mgid0);
                }
                spin_unlock_irq(&group->port->lock);
        }
        mcast_work_handler(&group->work);
}

static void leave_handler(int status, struct ib_sa_mcmember_rec *rec,
                          void *context)
{
        struct mcast_group *group = context;

        if (status && group->retries > 0 &&
            !send_leave(group, group->leave_state))
                group->retries--;
        else
                mcast_work_handler(&group->work);
}

static struct mcast_group *acquire_group(struct mcast_port *port,
                                         union ib_gid *mgid, gfp_t gfp_mask)
{
        struct mcast_group *group, *cur_group;
        unsigned long flags;
        int is_mgid0;

        is_mgid0 = !memcmp(&mgid0, mgid, sizeof mgid0);
        if (!is_mgid0) {
                spin_lock_irqsave(&port->lock, flags);
                group = mcast_find(port, mgid);
                if (group)
                        goto found;
                spin_unlock_irqrestore(&port->lock, flags);
        }

        group = kzalloc_obj(*group, gfp_mask);
        if (!group)
                return NULL;

        group->retries = 3;
        group->port = port;
        group->rec.mgid = *mgid;
        group->pkey_index = MCAST_INVALID_PKEY_INDEX;
        INIT_LIST_HEAD(&group->pending_list);
        INIT_LIST_HEAD(&group->active_list);
        INIT_WORK(&group->work, mcast_work_handler);
        spin_lock_init(&group->lock);

        spin_lock_irqsave(&port->lock, flags);
        cur_group = mcast_insert(port, group, is_mgid0);
        if (cur_group) {
                kfree(group);
                group = cur_group;
        } else
                refcount_inc(&port->refcount);
found:
        atomic_inc(&group->refcount);
        spin_unlock_irqrestore(&port->lock, flags);
        return group;
}

/*
 * We serialize all join requests to a single group to make our lives much
 * easier.  Otherwise, two users could try to join the same group
 * simultaneously, with different configurations, one could leave while the
 * join is in progress, etc., which makes locking around error recovery
 * difficult.
 */
struct ib_sa_multicast *
ib_sa_join_multicast(struct ib_sa_client *client,
                     struct ib_device *device, u32 port_num,
                     struct ib_sa_mcmember_rec *rec,
                     ib_sa_comp_mask comp_mask, gfp_t gfp_mask,
                     int (*callback)(int status,
                                     struct ib_sa_multicast *multicast),
                     void *context)
{
        struct mcast_device *dev;
        struct mcast_member *member;
        struct ib_sa_multicast *multicast;
        int ret;

        dev = ib_get_client_data(device, &mcast_client);
        if (!dev)
                return ERR_PTR(-ENODEV);

        member = kmalloc_obj(*member, gfp_mask);
        if (!member)
                return ERR_PTR(-ENOMEM);

        ib_sa_client_get(client);
        member->client = client;
        member->multicast.rec = *rec;
        member->multicast.comp_mask = comp_mask;
        member->multicast.callback = callback;
        member->multicast.context = context;
        init_completion(&member->comp);
        refcount_set(&member->refcount, 1);
        member->state = MCAST_JOINING;

        member->group = acquire_group(&dev->port[port_num - dev->start_port],
                                      &rec->mgid, gfp_mask);
        if (!member->group) {
                ret = -ENOMEM;
                goto err;
        }

        /*
         * The user will get the multicast structure in their callback.  They
         * could then free the multicast structure before we can return from
         * this routine.  So we save the pointer to return before queuing
         * any callback.
         */
        multicast = &member->multicast;
        queue_join(member);
        return multicast;

err:
        ib_sa_client_put(client);
        kfree(member);
        return ERR_PTR(ret);
}
EXPORT_SYMBOL(ib_sa_join_multicast);

void ib_sa_free_multicast(struct ib_sa_multicast *multicast)
{
        struct mcast_member *member;
        struct mcast_group *group;

        member = container_of(multicast, struct mcast_member, multicast);
        group = member->group;

        spin_lock_irq(&group->lock);
        if (member->state == MCAST_MEMBER)
                adjust_membership(group, multicast->rec.join_state, -1);

        list_del_init(&member->list);

        if (group->state == MCAST_IDLE) {
                group->state = MCAST_BUSY;
                spin_unlock_irq(&group->lock);
                /* Continue to hold reference on group until callback */
                queue_work(mcast_wq, &group->work);
        } else {
                spin_unlock_irq(&group->lock);
                release_group(group);
        }

        deref_member(member);
        wait_for_completion(&member->comp);
        ib_sa_client_put(member->client);
        kfree(member);
}
EXPORT_SYMBOL(ib_sa_free_multicast);

int ib_sa_get_mcmember_rec(struct ib_device *device, u32 port_num,
                           union ib_gid *mgid, struct ib_sa_mcmember_rec *rec)
{
        struct mcast_device *dev;
        struct mcast_port *port;
        struct mcast_group *group;
        unsigned long flags;
        int ret = 0;

        dev = ib_get_client_data(device, &mcast_client);
        if (!dev)
                return -ENODEV;

        port = &dev->port[port_num - dev->start_port];
        spin_lock_irqsave(&port->lock, flags);
        group = mcast_find(port, mgid);
        if (group)
                *rec = group->rec;
        else
                ret = -EADDRNOTAVAIL;
        spin_unlock_irqrestore(&port->lock, flags);

        return ret;
}
EXPORT_SYMBOL(ib_sa_get_mcmember_rec);

/**
 * ib_init_ah_from_mcmember - Initialize AH attribute from multicast
 * member record and gid of the device.
 * @device:        RDMA device
 * @port_num:        Port of the rdma device to consider
 * @rec:        Multicast member record to use
 * @ndev:        Optional netdevice, applicable only for RoCE
 * @gid_type:        GID type to consider
 * @ah_attr:        AH attribute to fillup on successful completion
 *
 * ib_init_ah_from_mcmember() initializes AH attribute based on multicast
 * member record and other device properties. On success the caller is
 * responsible to call rdma_destroy_ah_attr on the ah_attr. Returns 0 on
 * success or appropriate error code.
 *
 */
int ib_init_ah_from_mcmember(struct ib_device *device, u32 port_num,
                             struct ib_sa_mcmember_rec *rec,
                             struct net_device *ndev,
                             enum ib_gid_type gid_type,
                             struct rdma_ah_attr *ah_attr)
{
        const struct ib_gid_attr *sgid_attr;

        /* GID table is not based on the netdevice for IB link layer,
         * so ignore ndev during search.
         */
        if (rdma_protocol_ib(device, port_num))
                ndev = NULL;
        else if (!rdma_protocol_roce(device, port_num))
                return -EINVAL;

        sgid_attr = rdma_find_gid_by_port(device, &rec->port_gid,
                                          gid_type, port_num, ndev);
        if (IS_ERR(sgid_attr))
                return PTR_ERR(sgid_attr);

        memset(ah_attr, 0, sizeof(*ah_attr));
        ah_attr->type = rdma_ah_find_type(device, port_num);

        rdma_ah_set_dlid(ah_attr, be16_to_cpu(rec->mlid));
        rdma_ah_set_sl(ah_attr, rec->sl);
        rdma_ah_set_port_num(ah_attr, port_num);
        rdma_ah_set_static_rate(ah_attr, rec->rate);
        rdma_move_grh_sgid_attr(ah_attr, &rec->mgid,
                                be32_to_cpu(rec->flow_label),
                                rec->hop_limit,        rec->traffic_class,
                                sgid_attr);
        return 0;
}
EXPORT_SYMBOL(ib_init_ah_from_mcmember);

static void mcast_groups_event(struct mcast_port *port,
                               enum mcast_group_state state)
{
        struct mcast_group *group;
        struct rb_node *node;
        unsigned long flags;

        spin_lock_irqsave(&port->lock, flags);
        for (node = rb_first(&port->table); node; node = rb_next(node)) {
                group = rb_entry(node, struct mcast_group, node);
                spin_lock(&group->lock);
                if (group->state == MCAST_IDLE) {
                        atomic_inc(&group->refcount);
                        queue_work(mcast_wq, &group->work);
                }
                if (group->state != MCAST_GROUP_ERROR)
                        group->state = state;
                spin_unlock(&group->lock);
        }
        spin_unlock_irqrestore(&port->lock, flags);
}

static void mcast_event_handler(struct ib_event_handler *handler,
                                struct ib_event *event)
{
        struct mcast_device *dev;
        int index;

        dev = container_of(handler, struct mcast_device, event_handler);
        if (!rdma_cap_ib_mcast(dev->device, event->element.port_num))
                return;

        index = event->element.port_num - dev->start_port;

        switch (event->event) {
        case IB_EVENT_PORT_ERR:
        case IB_EVENT_LID_CHANGE:
        case IB_EVENT_CLIENT_REREGISTER:
                mcast_groups_event(&dev->port[index], MCAST_GROUP_ERROR);
                break;
        case IB_EVENT_PKEY_CHANGE:
                mcast_groups_event(&dev->port[index], MCAST_PKEY_EVENT);
                break;
        default:
                break;
        }
}

static int mcast_add_one(struct ib_device *device)
{
        struct mcast_device *dev;
        struct mcast_port *port;
        int i;
        int count = 0;

        dev = kmalloc_flex(*dev, port, device->phys_port_cnt);
        if (!dev)
                return -ENOMEM;

        dev->start_port = rdma_start_port(device);
        dev->end_port = rdma_end_port(device);

        for (i = 0; i <= dev->end_port - dev->start_port; i++) {
                if (!rdma_cap_ib_mcast(device, dev->start_port + i))
                        continue;
                port = &dev->port[i];
                port->dev = dev;
                port->port_num = dev->start_port + i;
                spin_lock_init(&port->lock);
                port->table = RB_ROOT;
                init_completion(&port->comp);
                refcount_set(&port->refcount, 1);
                ++count;
        }

        if (!count) {
                kfree(dev);
                return -EOPNOTSUPP;
        }

        dev->device = device;
        ib_set_client_data(device, &mcast_client, dev);

        INIT_IB_EVENT_HANDLER(&dev->event_handler, device, mcast_event_handler);
        ib_register_event_handler(&dev->event_handler);
        return 0;
}

static void mcast_remove_one(struct ib_device *device, void *client_data)
{
        struct mcast_device *dev = client_data;
        struct mcast_port *port;
        int i;

        ib_unregister_event_handler(&dev->event_handler);
        flush_workqueue(mcast_wq);

        for (i = 0; i <= dev->end_port - dev->start_port; i++) {
                if (rdma_cap_ib_mcast(device, dev->start_port + i)) {
                        port = &dev->port[i];
                        deref_port(port);
                        wait_for_completion(&port->comp);
                }
        }

        kfree(dev);
}

int mcast_init(void)
{
        int ret;

        mcast_wq = alloc_ordered_workqueue("ib_mcast", WQ_MEM_RECLAIM);
        if (!mcast_wq)
                return -ENOMEM;

        ib_sa_register_client(&sa_client);

        ret = ib_register_client(&mcast_client);
        if (ret)
                goto err;
        return 0;

err:
        ib_sa_unregister_client(&sa_client);
        destroy_workqueue(mcast_wq);
        return ret;
}

void mcast_cleanup(void)
{
        ib_unregister_client(&mcast_client);
        ib_sa_unregister_client(&sa_client);
        destroy_workqueue(mcast_wq);
}






















   32 






















   32 

   32 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
// SPDX-License-Identifier: GPL-2.0-or-later
/* mpihelp-mul_3.c  -  MPI helper functions
 * Copyright (C) 1994, 1996, 1997, 1998, 2001 Free Software Foundation, Inc.
 *
 * This file is part of GnuPG.
 *
 * Note: This code is heavily based on the GNU MP Library.
 *         Actually it's the same code with only minor changes in the
 *         way the data is stored; this is to support the abstraction
 *         of an optional secure memory allocation which may be used
 *         to avoid revealing of sensitive data due to paging etc.
 *         The GNU MP Library itself is published under the LGPL;
 *         however I decided to publish this code under the plain GPL.
 */

#include "mpi-internal.h"
#include "longlong.h"

mpi_limb_t
mpihelp_submul_1(mpi_ptr_t res_ptr, mpi_ptr_t s1_ptr,
                 mpi_size_t s1_size, mpi_limb_t s2_limb)
{
        mpi_limb_t cy_limb;
        mpi_size_t j;
        mpi_limb_t prod_high, prod_low;
        mpi_limb_t x;

        /* The loop counter and index J goes from -SIZE to -1.  This way
         * the loop becomes faster.  */
        j = -s1_size;
        res_ptr -= j;
        s1_ptr -= j;

        cy_limb = 0;
        do {
                umul_ppmm(prod_high, prod_low, s1_ptr[j], s2_limb);

                prod_low += cy_limb;
                cy_limb = (prod_low < cy_limb ? 1 : 0) + prod_high;

                x = res_ptr[j];
                prod_low = x - prod_low;
                cy_limb += prod_low > x ? 1 : 0;
                res_ptr[j] = prod_low;
        } while (++j);

        return cy_limb;
}

















































































   52 










   52 





















































































































































































































































































    2 
    2 



    5 


















    6 

























































    6 



































   16 


   15 
















   10 
















   10 



















   10 















   10 




















   10 



   10 









































   10 



   10 


   10 













   10 









































































   10 





   10 



   10 



















   10 



























   10 










   10 


   10 






   10 

   10 

   10 


   10 







   10 












   10 



















































   10 
















   10 

















   10 

   10 





   10 



































































































   10 



    1 
   10 




   10 


   10 





















































































    2 











    9 





   20 




































   19 
   20 














































   20 
   20 








   20 

















   19 
    9 
   19 





































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 1991, 1992 Linus Torvalds
 * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
 * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
 *        -  July2000
 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
 */

/*
 * This handles all read/write requests to block devices
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/blk-pm.h>
#include <linux/blk-integrity.h>
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/kernel_stat.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/completion.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/writeback.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/fault-inject.h>
#include <linux/list_sort.h>
#include <linux/delay.h>
#include <linux/ratelimit.h>
#include <linux/pm_runtime.h>
#include <linux/t10-pi.h>
#include <linux/debugfs.h>
#include <linux/bpf.h>
#include <linux/part_stat.h>
#include <linux/sched/sysctl.h>
#include <linux/blk-crypto.h>

#define CREATE_TRACE_POINTS
#include <trace/events/block.h>

#include "blk.h"
#include "blk-mq-sched.h"
#include "blk-pm.h"
#include "blk-cgroup.h"
#include "blk-throttle.h"
#include "blk-ioprio.h"

struct dentry *blk_debugfs_root;

EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert);

static DEFINE_IDA(blk_queue_ida);

/*
 * For queue allocation
 */
static struct kmem_cache *blk_requestq_cachep;

/*
 * Controlling structure to kblockd
 */
static struct workqueue_struct *kblockd_workqueue;

/**
 * blk_queue_flag_set - atomically set a queue flag
 * @flag: flag to be set
 * @q: request queue
 */
void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
{
        set_bit(flag, &q->queue_flags);
}
EXPORT_SYMBOL(blk_queue_flag_set);

/**
 * blk_queue_flag_clear - atomically clear a queue flag
 * @flag: flag to be cleared
 * @q: request queue
 */
void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
{
        clear_bit(flag, &q->queue_flags);
}
EXPORT_SYMBOL(blk_queue_flag_clear);

#define REQ_OP_NAME(name) [REQ_OP_##name] = #name
static const char *const blk_op_name[] = {
        REQ_OP_NAME(READ),
        REQ_OP_NAME(WRITE),
        REQ_OP_NAME(FLUSH),
        REQ_OP_NAME(DISCARD),
        REQ_OP_NAME(SECURE_ERASE),
        REQ_OP_NAME(ZONE_RESET),
        REQ_OP_NAME(ZONE_RESET_ALL),
        REQ_OP_NAME(ZONE_OPEN),
        REQ_OP_NAME(ZONE_CLOSE),
        REQ_OP_NAME(ZONE_FINISH),
        REQ_OP_NAME(ZONE_APPEND),
        REQ_OP_NAME(WRITE_ZEROES),
        REQ_OP_NAME(DRV_IN),
        REQ_OP_NAME(DRV_OUT),
};
#undef REQ_OP_NAME

/**
 * blk_op_str - Return the string "name" for an operation REQ_OP_name.
 * @op: a request operation.
 *
 * Convert a request operation REQ_OP_name into the string "name". Useful for
 * debugging and tracing BIOs and requests. For an invalid request operation
 * code, the string "UNKNOWN" is returned.
 */
inline const char *blk_op_str(enum req_op op)
{
        const char *op_str = "UNKNOWN";

        if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op])
                op_str = blk_op_name[op];

        return op_str;
}
EXPORT_SYMBOL_GPL(blk_op_str);

static const struct {
        int                errno;
        const char        *name;
} blk_errors[] = {
        [BLK_STS_OK]                = { 0,                "" },
        [BLK_STS_NOTSUPP]        = { -EOPNOTSUPP, "operation not supported" },
        [BLK_STS_TIMEOUT]        = { -ETIMEDOUT,        "timeout" },
        [BLK_STS_NOSPC]                = { -ENOSPC,        "critical space allocation" },
        [BLK_STS_TRANSPORT]        = { -ENOLINK,        "recoverable transport" },
        [BLK_STS_TARGET]        = { -EREMOTEIO,        "critical target" },
        [BLK_STS_RESV_CONFLICT]        = { -EBADE,        "reservation conflict" },
        [BLK_STS_MEDIUM]        = { -ENODATA,        "critical medium" },
        [BLK_STS_PROTECTION]        = { -EILSEQ,        "protection" },
        [BLK_STS_RESOURCE]        = { -ENOMEM,        "kernel resource" },
        [BLK_STS_DEV_RESOURCE]        = { -EBUSY,        "device resource" },
        [BLK_STS_AGAIN]                = { -EAGAIN,        "nonblocking retry" },
        [BLK_STS_OFFLINE]        = { -ENODEV,        "device offline" },

        /* device mapper special case, should not leak out: */
        [BLK_STS_DM_REQUEUE]        = { -EREMCHG, "dm internal retry" },

        /* zone device specific errors */
        [BLK_STS_ZONE_OPEN_RESOURCE]        = { -ETOOMANYREFS, "open zones exceeded" },
        [BLK_STS_ZONE_ACTIVE_RESOURCE]        = { -EOVERFLOW, "active zones exceeded" },

        /* Command duration limit device-side timeout */
        [BLK_STS_DURATION_LIMIT]        = { -ETIME, "duration limit exceeded" },

        [BLK_STS_INVAL]                = { -EINVAL,        "invalid" },

        /* everything else not covered above: */
        [BLK_STS_IOERR]                = { -EIO,        "I/O" },
};

blk_status_t errno_to_blk_status(int errno)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
                if (blk_errors[i].errno == errno)
                        return (__force blk_status_t)i;
        }

        return BLK_STS_IOERR;
}
EXPORT_SYMBOL_GPL(errno_to_blk_status);

int blk_status_to_errno(blk_status_t status)
{
        int idx = (__force int)status;

        if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
                return -EIO;
        return blk_errors[idx].errno;
}
EXPORT_SYMBOL_GPL(blk_status_to_errno);

const char *blk_status_to_str(blk_status_t status)
{
        int idx = (__force int)status;

        if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
                return "<null>";
        return blk_errors[idx].name;
}
EXPORT_SYMBOL_GPL(blk_status_to_str);

/**
 * blk_sync_queue - cancel any pending callbacks on a queue
 * @q: the queue
 *
 * Description:
 *     The block layer may perform asynchronous callback activity
 *     on a queue, such as calling the unplug function after a timeout.
 *     A block device may call blk_sync_queue to ensure that any
 *     such activity is cancelled, thus allowing it to release resources
 *     that the callbacks might use. The caller must already have made sure
 *     that its ->submit_bio will not re-add plugging prior to calling
 *     this function.
 *
 *     This function does not cancel any asynchronous activity arising
 *     out of elevator or throttling code. That would require elevator_exit()
 *     and blkcg_exit_queue() to be called with queue lock initialized.
 *
 */
void blk_sync_queue(struct request_queue *q)
{
        timer_delete_sync(&q->timeout);
        cancel_work_sync(&q->timeout_work);
}
EXPORT_SYMBOL(blk_sync_queue);

/**
 * blk_set_pm_only - increment pm_only counter
 * @q: request queue pointer
 */
void blk_set_pm_only(struct request_queue *q)
{
        atomic_inc(&q->pm_only);
}
EXPORT_SYMBOL_GPL(blk_set_pm_only);

void blk_clear_pm_only(struct request_queue *q)
{
        int pm_only;

        pm_only = atomic_dec_return(&q->pm_only);
        WARN_ON_ONCE(pm_only < 0);
        if (pm_only == 0)
                wake_up_all(&q->mq_freeze_wq);
}
EXPORT_SYMBOL_GPL(blk_clear_pm_only);

static void blk_free_queue_rcu(struct rcu_head *rcu_head)
{
        struct request_queue *q = container_of(rcu_head,
                        struct request_queue, rcu_head);

        percpu_ref_exit(&q->q_usage_counter);
        kmem_cache_free(blk_requestq_cachep, q);
}

static void blk_free_queue(struct request_queue *q)
{
        blk_free_queue_stats(q->stats);
        if (queue_is_mq(q))
                blk_mq_release(q);

        ida_free(&blk_queue_ida, q->id);
        lockdep_unregister_key(&q->io_lock_cls_key);
        lockdep_unregister_key(&q->q_lock_cls_key);
        call_rcu(&q->rcu_head, blk_free_queue_rcu);
}

/**
 * blk_put_queue - decrement the request_queue refcount
 * @q: the request_queue structure to decrement the refcount for
 *
 * Decrements the refcount of the request_queue and free it when the refcount
 * reaches 0.
 */
void blk_put_queue(struct request_queue *q)
{
        if (refcount_dec_and_test(&q->refs))
                blk_free_queue(q);
}
EXPORT_SYMBOL(blk_put_queue);

bool blk_queue_start_drain(struct request_queue *q)
{
        /*
         * When queue DYING flag is set, we need to block new req
         * entering queue, so we call blk_freeze_queue_start() to
         * prevent I/O from crossing blk_queue_enter().
         */
        bool freeze = __blk_freeze_queue_start(q, current);
        if (queue_is_mq(q))
                blk_mq_wake_waiters(q);
        /* Make blk_queue_enter() reexamine the DYING flag. */
        wake_up_all(&q->mq_freeze_wq);

        return freeze;
}

/**
 * blk_queue_enter() - try to increase q->q_usage_counter
 * @q: request queue pointer
 * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PM
 */
int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
{
        const bool pm = flags & BLK_MQ_REQ_PM;

        while (!blk_try_enter_queue(q, pm)) {
                if (flags & BLK_MQ_REQ_NOWAIT)
                        return -EAGAIN;

                /*
                 * read pair of barrier in blk_freeze_queue_start(), we need to
                 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and
                 * reading .mq_freeze_depth or queue dying flag, otherwise the
                 * following wait may never return if the two reads are
                 * reordered.
                 */
                smp_rmb();
                wait_event(q->mq_freeze_wq,
                           (!q->mq_freeze_depth &&
                            blk_pm_resume_queue(pm, q)) ||
                           blk_queue_dying(q));
                if (blk_queue_dying(q))
                        return -ENODEV;
        }

        rwsem_acquire_read(&q->q_lockdep_map, 0, 0, _RET_IP_);
        rwsem_release(&q->q_lockdep_map, _RET_IP_);
        return 0;
}

int __bio_queue_enter(struct request_queue *q, struct bio *bio)
{
        while (!blk_try_enter_queue(q, false)) {
                struct gendisk *disk = bio->bi_bdev->bd_disk;

                if (bio->bi_opf & REQ_NOWAIT) {
                        if (test_bit(GD_DEAD, &disk->state))
                                goto dead;
                        bio_wouldblock_error(bio);
                        return -EAGAIN;
                }

                /*
                 * read pair of barrier in blk_freeze_queue_start(), we need to
                 * order reading __PERCPU_REF_DEAD flag of .q_usage_counter and
                 * reading .mq_freeze_depth or queue dying flag, otherwise the
                 * following wait may never return if the two reads are
                 * reordered.
                 */
                smp_rmb();
                wait_event(q->mq_freeze_wq,
                           (!q->mq_freeze_depth &&
                            blk_pm_resume_queue(false, q)) ||
                           test_bit(GD_DEAD, &disk->state));
                if (test_bit(GD_DEAD, &disk->state))
                        goto dead;
        }

        rwsem_acquire_read(&q->io_lockdep_map, 0, 0, _RET_IP_);
        rwsem_release(&q->io_lockdep_map, _RET_IP_);
        return 0;
dead:
        bio_io_error(bio);
        return -ENODEV;
}

void blk_queue_exit(struct request_queue *q)
{
        percpu_ref_put(&q->q_usage_counter);
}

static void blk_queue_usage_counter_release(struct percpu_ref *ref)
{
        struct request_queue *q =
                container_of(ref, struct request_queue, q_usage_counter);

        wake_up_all(&q->mq_freeze_wq);
}

static void blk_rq_timed_out_timer(struct timer_list *t)
{
        struct request_queue *q = timer_container_of(q, t, timeout);

        kblockd_schedule_work(&q->timeout_work);
}

static void blk_timeout_work(struct work_struct *work)
{
}

struct request_queue *blk_alloc_queue(struct queue_limits *lim, int node_id)
{
        struct request_queue *q;
        int error;

        q = kmem_cache_alloc_node(blk_requestq_cachep, GFP_KERNEL | __GFP_ZERO,
                                  node_id);
        if (!q)
                return ERR_PTR(-ENOMEM);

        q->last_merge = NULL;

        q->id = ida_alloc(&blk_queue_ida, GFP_KERNEL);
        if (q->id < 0) {
                error = q->id;
                goto fail_q;
        }

        q->stats = blk_alloc_queue_stats();
        if (!q->stats) {
                error = -ENOMEM;
                goto fail_id;
        }

        error = blk_set_default_limits(lim);
        if (error)
                goto fail_stats;
        q->limits = *lim;

        q->node = node_id;

        atomic_set(&q->nr_active_requests_shared_tags, 0);

        timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
        INIT_WORK(&q->timeout_work, blk_timeout_work);
        INIT_LIST_HEAD(&q->icq_list);

        refcount_set(&q->refs, 1);
        mutex_init(&q->debugfs_mutex);
        mutex_init(&q->elevator_lock);
        mutex_init(&q->sysfs_lock);
        mutex_init(&q->limits_lock);
        mutex_init(&q->rq_qos_mutex);
        spin_lock_init(&q->queue_lock);

        init_waitqueue_head(&q->mq_freeze_wq);
        mutex_init(&q->mq_freeze_lock);

        blkg_init_queue(q);

        /*
         * Init percpu_ref in atomic mode so that it's faster to shutdown.
         * See blk_register_queue() for details.
         */
        error = percpu_ref_init(&q->q_usage_counter,
                                blk_queue_usage_counter_release,
                                PERCPU_REF_INIT_ATOMIC, GFP_KERNEL);
        if (error)
                goto fail_stats;
        lockdep_register_key(&q->io_lock_cls_key);
        lockdep_register_key(&q->q_lock_cls_key);
        lockdep_init_map(&q->io_lockdep_map, "&q->q_usage_counter(io)",
                         &q->io_lock_cls_key, 0);
        lockdep_init_map(&q->q_lockdep_map, "&q->q_usage_counter(queue)",
                         &q->q_lock_cls_key, 0);

        /* Teach lockdep about lock ordering (reclaim WRT queue freeze lock). */
        fs_reclaim_acquire(GFP_KERNEL);
        rwsem_acquire_read(&q->io_lockdep_map, 0, 0, _RET_IP_);
        rwsem_release(&q->io_lockdep_map, _RET_IP_);
        fs_reclaim_release(GFP_KERNEL);

        q->nr_requests = BLKDEV_DEFAULT_RQ;
        q->async_depth = BLKDEV_DEFAULT_RQ;

        return q;

fail_stats:
        blk_free_queue_stats(q->stats);
fail_id:
        ida_free(&blk_queue_ida, q->id);
fail_q:
        kmem_cache_free(blk_requestq_cachep, q);
        return ERR_PTR(error);
}

/**
 * blk_get_queue - increment the request_queue refcount
 * @q: the request_queue structure to increment the refcount for
 *
 * Increment the refcount of the request_queue kobject.
 *
 * Context: Any context.
 */
bool blk_get_queue(struct request_queue *q)
{
        if (unlikely(blk_queue_dying(q)))
                return false;
        refcount_inc(&q->refs);
        return true;
}
EXPORT_SYMBOL(blk_get_queue);

#ifdef CONFIG_FAIL_MAKE_REQUEST

static DECLARE_FAULT_ATTR(fail_make_request);

static int __init setup_fail_make_request(char *str)
{
        return setup_fault_attr(&fail_make_request, str);
}
__setup("fail_make_request=", setup_fail_make_request);

bool should_fail_request(struct block_device *part, unsigned int bytes)
{
        return bdev_test_flag(part, BD_MAKE_IT_FAIL) &&
               should_fail(&fail_make_request, bytes);
}

static int __init fail_make_request_debugfs(void)
{
        struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
                                                NULL, &fail_make_request);

        return PTR_ERR_OR_ZERO(dir);
}

late_initcall(fail_make_request_debugfs);
#endif /* CONFIG_FAIL_MAKE_REQUEST */

static inline void bio_check_ro(struct bio *bio)
{
        if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) {
                if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
                        return;

                if (bdev_test_flag(bio->bi_bdev, BD_RO_WARNED))
                        return;

                bdev_set_flag(bio->bi_bdev, BD_RO_WARNED);

                /*
                 * Use ioctl to set underlying disk of raid/dm to read-only
                 * will trigger this.
                 */
                pr_warn("Trying to write to read-only block-device %pg\n",
                        bio->bi_bdev);
        }
}

int should_fail_bio(struct bio *bio)
{
        if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size))
                return -EIO;
        return 0;
}
ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);

/*
 * Check whether this bio extends beyond the end of the device or partition.
 * This may well happen - the kernel calls bread() without checking the size of
 * the device, e.g., when mounting a file system.
 */
static inline int bio_check_eod(struct bio *bio)
{
        sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
        unsigned int nr_sectors = bio_sectors(bio);

        if (nr_sectors &&
            (nr_sectors > maxsector ||
             bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
                if (!maxsector)
                        return -EIO;
                pr_info_ratelimited("%s: attempt to access beyond end of device\n"
                                    "%pg: rw=%d, sector=%llu, nr_sectors = %u limit=%llu\n",
                                    current->comm, bio->bi_bdev, bio->bi_opf,
                                    bio->bi_iter.bi_sector, nr_sectors, maxsector);
                return -EIO;
        }
        return 0;
}

/*
 * Remap block n of partition p to block n+start(p) of the disk.
 */
static int blk_partition_remap(struct bio *bio)
{
        struct block_device *p = bio->bi_bdev;

        if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
                return -EIO;
        if (bio_sectors(bio)) {
                bio->bi_iter.bi_sector += p->bd_start_sect;
                trace_block_bio_remap(bio, p->bd_dev,
                                      bio->bi_iter.bi_sector -
                                      p->bd_start_sect);
        }
        bio_set_flag(bio, BIO_REMAPPED);
        return 0;
}

/*
 * Check write append to a zoned block device.
 */
static inline blk_status_t blk_check_zone_append(struct request_queue *q,
                                                 struct bio *bio)
{
        int nr_sectors = bio_sectors(bio);

        /* Only applicable to zoned block devices */
        if (!bdev_is_zoned(bio->bi_bdev))
                return BLK_STS_NOTSUPP;

        /* The bio sector must point to the start of a sequential zone */
        if (!bdev_is_zone_start(bio->bi_bdev, bio->bi_iter.bi_sector))
                return BLK_STS_IOERR;

        /*
         * Not allowed to cross zone boundaries. Otherwise, the BIO will be
         * split and could result in non-contiguous sectors being written in
         * different zones.
         */
        if (nr_sectors > q->limits.chunk_sectors)
                return BLK_STS_IOERR;

        /* Make sure the BIO is small enough and will not get split */
        if (nr_sectors > q->limits.max_zone_append_sectors)
                return BLK_STS_IOERR;

        bio->bi_opf |= REQ_NOMERGE;

        return BLK_STS_OK;
}

static void __submit_bio(struct bio *bio)
{
        /* If plug is not used, add new plug here to cache nsecs time. */
        struct blk_plug plug;

        blk_start_plug(&plug);

        if (!bdev_test_flag(bio->bi_bdev, BD_HAS_SUBMIT_BIO)) {
                blk_mq_submit_bio(bio);
        } else if (likely(bio_queue_enter(bio) == 0)) {
                struct gendisk *disk = bio->bi_bdev->bd_disk;
        
                if ((bio->bi_opf & REQ_POLLED) &&
                    !(disk->queue->limits.features & BLK_FEAT_POLL)) {
                        bio->bi_status = BLK_STS_NOTSUPP;
                        bio_endio(bio);
                } else {
                        disk->fops->submit_bio(bio);
                }
                blk_queue_exit(disk->queue);
        }

        blk_finish_plug(&plug);
}

/*
 * The loop in this function may be a bit non-obvious, and so deserves some
 * explanation:
 *
 *  - Before entering the loop, bio->bi_next is NULL (as all callers ensure
 *    that), so we have a list with a single bio.
 *  - We pretend that we have just taken it off a longer list, so we assign
 *    bio_list to a pointer to the bio_list_on_stack, thus initialising the
 *    bio_list of new bios to be added.  ->submit_bio() may indeed add some more
 *    bios through a recursive call to submit_bio_noacct.  If it did, we find a
 *    non-NULL value in bio_list and re-enter the loop from the top.
 *  - In this case we really did just take the bio off the top of the list (no
 *    pretending) and so remove it from bio_list, and call into ->submit_bio()
 *    again.
 *
 * bio_list_on_stack[0] contains bios submitted by the current ->submit_bio.
 * bio_list_on_stack[1] contains bios that were submitted before the current
 *        ->submit_bio(), but that haven't been processed yet.
 */
static void __submit_bio_noacct(struct bio *bio)
{
        struct bio_list bio_list_on_stack[2];

        BUG_ON(bio->bi_next);

        bio_list_init(&bio_list_on_stack[0]);
        current->bio_list = bio_list_on_stack;

        do {
                struct request_queue *q = bdev_get_queue(bio->bi_bdev);
                struct bio_list lower, same;

                /*
                 * Create a fresh bio_list for all subordinate requests.
                 */
                bio_list_on_stack[1] = bio_list_on_stack[0];
                bio_list_init(&bio_list_on_stack[0]);

                __submit_bio(bio);

                /*
                 * Sort new bios into those for a lower level and those for the
                 * same level.
                 */
                bio_list_init(&lower);
                bio_list_init(&same);
                while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
                        if (q == bdev_get_queue(bio->bi_bdev))
                                bio_list_add(&same, bio);
                        else
                                bio_list_add(&lower, bio);

                /*
                 * Now assemble so we handle the lowest level first.
                 */
                bio_list_merge(&bio_list_on_stack[0], &lower);
                bio_list_merge(&bio_list_on_stack[0], &same);
                bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
        } while ((bio = bio_list_pop(&bio_list_on_stack[0])));

        current->bio_list = NULL;
}

static void __submit_bio_noacct_mq(struct bio *bio)
{
        struct bio_list bio_list[2] = { };

        current->bio_list = bio_list;

        do {
                __submit_bio(bio);
        } while ((bio = bio_list_pop(&bio_list[0])));

        current->bio_list = NULL;
}

void submit_bio_noacct_nocheck(struct bio *bio, bool split)
{
        blk_cgroup_bio_start(bio);

        if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
                trace_block_bio_queue(bio);
                /*
                 * Now that enqueuing has been traced, we need to trace
                 * completion as well.
                 */
                bio_set_flag(bio, BIO_TRACE_COMPLETION);
        }

        /*
         * We only want one ->submit_bio to be active at a time, else stack
         * usage with stacked devices could be a problem.  Use current->bio_list
         * to collect a list of requests submitted by a ->submit_bio method
         * while it is active, and then process them after it returned.
         */
        if (current->bio_list) {
                if (split)
                        bio_list_add_head(&current->bio_list[0], bio);
                else
                        bio_list_add(&current->bio_list[0], bio);
        } else if (!bdev_test_flag(bio->bi_bdev, BD_HAS_SUBMIT_BIO)) {
                __submit_bio_noacct_mq(bio);
        } else {
                __submit_bio_noacct(bio);
        }
}

static blk_status_t blk_validate_atomic_write_op_size(struct request_queue *q,
                                                 struct bio *bio)
{
        if (bio->bi_iter.bi_size > queue_atomic_write_unit_max_bytes(q))
                return BLK_STS_INVAL;

        if (bio->bi_iter.bi_size % queue_atomic_write_unit_min_bytes(q))
                return BLK_STS_INVAL;

        return BLK_STS_OK;
}

/**
 * submit_bio_noacct - re-submit a bio to the block device layer for I/O
 * @bio:  The bio describing the location in memory and on the device.
 *
 * This is a version of submit_bio() that shall only be used for I/O that is
 * resubmitted to lower level drivers by stacking block drivers.  All file
 * systems and other upper level users of the block layer should use
 * submit_bio() instead.
 */
void submit_bio_noacct(struct bio *bio)
{
        struct block_device *bdev = bio->bi_bdev;
        struct request_queue *q = bdev_get_queue(bdev);
        blk_status_t status = BLK_STS_IOERR;

        might_sleep();

        /*
         * For a REQ_NOWAIT based request, return -EOPNOTSUPP
         * if queue does not support NOWAIT.
         */
        if ((bio->bi_opf & REQ_NOWAIT) && !bdev_nowait(bdev))
                goto not_supported;

        if (bio_has_crypt_ctx(bio)) {
                if (WARN_ON_ONCE(!bio_has_data(bio)))
                        goto end_io;
                if (!blk_crypto_supported(bio))
                        goto not_supported;
        }

        if (should_fail_bio(bio))
                goto end_io;
        bio_check_ro(bio);
        if (!bio_flagged(bio, BIO_REMAPPED)) {
                if (unlikely(bio_check_eod(bio)))
                        goto end_io;
                if (bdev_is_partition(bdev) &&
                    unlikely(blk_partition_remap(bio)))
                        goto end_io;
        }

        /*
         * Filter flush bio's early so that bio based drivers without flush
         * support don't have to worry about them.
         */
        if (op_is_flush(bio->bi_opf)) {
                if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_WRITE &&
                                 bio_op(bio) != REQ_OP_ZONE_APPEND))
                        goto end_io;
                if (!bdev_write_cache(bdev)) {
                        bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
                        if (!bio_sectors(bio)) {
                                status = BLK_STS_OK;
                                goto end_io;
                        }
                }
        }

        switch (bio_op(bio)) {
        case REQ_OP_READ:
                break;
        case REQ_OP_WRITE:
                if (bio->bi_opf & REQ_ATOMIC) {
                        status = blk_validate_atomic_write_op_size(q, bio);
                        if (status != BLK_STS_OK)
                                goto end_io;
                }
                break;
        case REQ_OP_FLUSH:
                /*
                 * REQ_OP_FLUSH can't be submitted through bios, it is only
                 * synthetized in struct request by the flush state machine.
                 */
                goto not_supported;
        case REQ_OP_DISCARD:
                if (!bdev_max_discard_sectors(bdev))
                        goto not_supported;
                break;
        case REQ_OP_SECURE_ERASE:
                if (!bdev_max_secure_erase_sectors(bdev))
                        goto not_supported;
                break;
        case REQ_OP_ZONE_APPEND:
                status = blk_check_zone_append(q, bio);
                if (status != BLK_STS_OK)
                        goto end_io;
                break;
        case REQ_OP_WRITE_ZEROES:
                if (!q->limits.max_write_zeroes_sectors)
                        goto not_supported;
                break;
        case REQ_OP_ZONE_RESET:
        case REQ_OP_ZONE_OPEN:
        case REQ_OP_ZONE_CLOSE:
        case REQ_OP_ZONE_FINISH:
        case REQ_OP_ZONE_RESET_ALL:
                if (!bdev_is_zoned(bio->bi_bdev))
                        goto not_supported;
                break;
        case REQ_OP_DRV_IN:
        case REQ_OP_DRV_OUT:
                /*
                 * Driver private operations are only used with passthrough
                 * requests.
                 */
                fallthrough;
        default:
                goto not_supported;
        }

        if (blk_throtl_bio(bio))
                return;
        submit_bio_noacct_nocheck(bio, false);
        return;

not_supported:
        status = BLK_STS_NOTSUPP;
end_io:
        bio->bi_status = status;
        bio_endio(bio);
}
EXPORT_SYMBOL(submit_bio_noacct);

static void bio_set_ioprio(struct bio *bio)
{
        /* Nobody set ioprio so far? Initialize it based on task's nice value */
        if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) == IOPRIO_CLASS_NONE)
                bio->bi_ioprio = get_current_ioprio();
        blkcg_set_ioprio(bio);
}

/**
 * submit_bio - submit a bio to the block device layer for I/O
 * @bio: The &struct bio which describes the I/O
 *
 * submit_bio() is used to submit I/O requests to block devices.  It is passed a
 * fully set up &struct bio that describes the I/O that needs to be done.  The
 * bio will be sent to the device described by the bi_bdev field.
 *
 * The success/failure status of the request, along with notification of
 * completion, is delivered asynchronously through the ->bi_end_io() callback
 * in @bio.  The bio must NOT be touched by the caller until ->bi_end_io() has
 * been called.
 */
void submit_bio(struct bio *bio)
{
        if (bio_op(bio) == REQ_OP_READ) {
                task_io_account_read(bio->bi_iter.bi_size);
                count_vm_events(PGPGIN, bio_sectors(bio));
        } else if (bio_op(bio) == REQ_OP_WRITE) {
                count_vm_events(PGPGOUT, bio_sectors(bio));
        }

        bio_set_ioprio(bio);
        submit_bio_noacct(bio);
}
EXPORT_SYMBOL(submit_bio);

/**
 * bio_poll - poll for BIO completions
 * @bio: bio to poll for
 * @iob: batches of IO
 * @flags: BLK_POLL_* flags that control the behavior
 *
 * Poll for completions on queue associated with the bio. Returns number of
 * completed entries found.
 *
 * Note: the caller must either be the context that submitted @bio, or
 * be in a RCU critical section to prevent freeing of @bio.
 */
int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags)
{
        blk_qc_t cookie = READ_ONCE(bio->bi_cookie);
        struct block_device *bdev;
        struct request_queue *q;
        int ret = 0;

        bdev = READ_ONCE(bio->bi_bdev);
        if (!bdev)
                return 0;

        q = bdev_get_queue(bdev);
        if (cookie == BLK_QC_T_NONE)
                return 0;

        blk_flush_plug(current->plug, false);

        /*
         * We need to be able to enter a frozen queue, similar to how
         * timeouts also need to do that. If that is blocked, then we can
         * have pending IO when a queue freeze is started, and then the
         * wait for the freeze to finish will wait for polled requests to
         * timeout as the poller is preventer from entering the queue and
         * completing them. As long as we prevent new IO from being queued,
         * that should be all that matters.
         */
        if (!percpu_ref_tryget(&q->q_usage_counter))
                return 0;
        if (queue_is_mq(q)) {
                ret = blk_mq_poll(q, cookie, iob, flags);
        } else {
                struct gendisk *disk = q->disk;

                if ((q->limits.features & BLK_FEAT_POLL) && disk &&
                    disk->fops->poll_bio)
                        ret = disk->fops->poll_bio(bio, iob, flags);
        }
        blk_queue_exit(q);
        return ret;
}
EXPORT_SYMBOL_GPL(bio_poll);

/*
 * Helper to implement file_operations.iopoll.  Requires the bio to be stored
 * in iocb->private, and cleared before freeing the bio.
 */
int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
                    unsigned int flags)
{
        struct bio *bio;
        int ret = 0;

        /*
         * Note: the bio cache only uses SLAB_TYPESAFE_BY_RCU, so bio can
         * point to a freshly allocated bio at this point.  If that happens
         * we have a few cases to consider:
         *
         *  1) the bio is being initialized and bi_bdev is NULL.  We can just
         *     simply nothing in this case
         *  2) the bio points to a not poll enabled device.  bio_poll will catch
         *     this and return 0
         *  3) the bio points to a poll capable device, including but not
         *     limited to the one that the original bio pointed to.  In this
         *     case we will call into the actual poll method and poll for I/O,
         *     even if we don't need to, but it won't cause harm either.
         *
         * For cases 2) and 3) above the RCU grace period ensures that bi_bdev
         * is still allocated. Because partitions hold a reference to the whole
         * device bdev and thus disk, the disk is also still valid.  Grabbing
         * a reference to the queue in bio_poll() ensures the hctxs and requests
         * are still valid as well.
         */
        rcu_read_lock();
        bio = READ_ONCE(kiocb->private);
        if (bio)
                ret = bio_poll(bio, iob, flags);
        rcu_read_unlock();

        return ret;
}
EXPORT_SYMBOL_GPL(iocb_bio_iopoll);

void update_io_ticks(struct block_device *part, unsigned long now, bool end)
{
        unsigned long stamp;
again:
        stamp = READ_ONCE(part->bd_stamp);
        if (unlikely(time_after(now, stamp)) &&
            likely(try_cmpxchg(&part->bd_stamp, &stamp, now)) &&
            (end || bdev_count_inflight(part)))
                __part_stat_add(part, io_ticks, now - stamp);

        if (bdev_is_partition(part)) {
                part = bdev_whole(part);
                goto again;
        }
}

unsigned long bdev_start_io_acct(struct block_device *bdev, enum req_op op,
                                 unsigned long start_time)
{
        part_stat_lock();
        update_io_ticks(bdev, start_time, false);
        part_stat_local_inc(bdev, in_flight[op_is_write(op)]);
        part_stat_unlock();

        return start_time;
}
EXPORT_SYMBOL(bdev_start_io_acct);

/**
 * bio_start_io_acct - start I/O accounting for bio based drivers
 * @bio:        bio to start account for
 *
 * Returns the start time that should be passed back to bio_end_io_acct().
 */
unsigned long bio_start_io_acct(struct bio *bio)
{
        return bdev_start_io_acct(bio->bi_bdev, bio_op(bio), jiffies);
}
EXPORT_SYMBOL_GPL(bio_start_io_acct);

void bdev_end_io_acct(struct block_device *bdev, enum req_op op,
                      unsigned int sectors, unsigned long start_time)
{
        const int sgrp = op_stat_group(op);
        unsigned long now = READ_ONCE(jiffies);
        unsigned long duration = now - start_time;

        part_stat_lock();
        update_io_ticks(bdev, now, true);
        part_stat_inc(bdev, ios[sgrp]);
        part_stat_add(bdev, sectors[sgrp], sectors);
        part_stat_add(bdev, nsecs[sgrp], jiffies_to_nsecs(duration));
        part_stat_local_dec(bdev, in_flight[op_is_write(op)]);
        part_stat_unlock();
}
EXPORT_SYMBOL(bdev_end_io_acct);

void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
                              struct block_device *orig_bdev)
{
        bdev_end_io_acct(orig_bdev, bio_op(bio), bio_sectors(bio), start_time);
}
EXPORT_SYMBOL_GPL(bio_end_io_acct_remapped);

/**
 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
 * @q : the queue of the device being checked
 *
 * Description:
 *    Check if underlying low-level drivers of a device are busy.
 *    If the drivers want to export their busy state, they must set own
 *    exporting function using blk_queue_lld_busy() first.
 *
 *    Basically, this function is used only by request stacking drivers
 *    to stop dispatching requests to underlying devices when underlying
 *    devices are busy.  This behavior helps more I/O merging on the queue
 *    of the request stacking driver and prevents I/O throughput regression
 *    on burst I/O load.
 *
 * Return:
 *    0 - Not busy (The request stacking driver should dispatch request)
 *    1 - Busy (The request stacking driver should stop dispatching request)
 */
int blk_lld_busy(struct request_queue *q)
{
        if (queue_is_mq(q) && q->mq_ops->busy)
                return q->mq_ops->busy(q);

        return 0;
}
EXPORT_SYMBOL_GPL(blk_lld_busy);

int kblockd_schedule_work(struct work_struct *work)
{
        return queue_work(kblockd_workqueue, work);
}
EXPORT_SYMBOL(kblockd_schedule_work);

int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
                                unsigned long delay)
{
        return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
}
EXPORT_SYMBOL(kblockd_mod_delayed_work_on);

void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios)
{
        struct task_struct *tsk = current;

        /*
         * If this is a nested plug, don't actually assign it.
         */
        if (tsk->plug)
                return;

        plug->cur_ktime = 0;
        rq_list_init(&plug->mq_list);
        rq_list_init(&plug->cached_rqs);
        plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);
        plug->rq_count = 0;
        plug->multiple_queues = false;
        plug->has_elevator = false;
        INIT_LIST_HEAD(&plug->cb_list);

        /*
         * Store ordering should not be needed here, since a potential
         * preempt will imply a full memory barrier
         */
        tsk->plug = plug;
}

/**
 * blk_start_plug - initialize blk_plug and track it inside the task_struct
 * @plug:        The &struct blk_plug that needs to be initialized
 *
 * Description:
 *   blk_start_plug() indicates to the block layer an intent by the caller
 *   to submit multiple I/O requests in a batch.  The block layer may use
 *   this hint to defer submitting I/Os from the caller until blk_finish_plug()
 *   is called.  However, the block layer may choose to submit requests
 *   before a call to blk_finish_plug() if the number of queued I/Os
 *   exceeds %BLK_MAX_REQUEST_COUNT, or if the size of the I/O is larger than
 *   %BLK_PLUG_FLUSH_SIZE.  The queued I/Os may also be submitted early if
 *   the task schedules (see below).
 *
 *   Tracking blk_plug inside the task_struct will help with auto-flushing the
 *   pending I/O should the task end up blocking between blk_start_plug() and
 *   blk_finish_plug(). This is important from a performance perspective, but
 *   also ensures that we don't deadlock. For instance, if the task is blocking
 *   for a memory allocation, memory reclaim could end up wanting to free a
 *   page belonging to that request that is currently residing in our private
 *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
 *   this kind of deadlock.
 */
void blk_start_plug(struct blk_plug *plug)
{
        blk_start_plug_nr_ios(plug, 1);
}
EXPORT_SYMBOL(blk_start_plug);

static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
{
        LIST_HEAD(callbacks);

        while (!list_empty(&plug->cb_list)) {
                list_splice_init(&plug->cb_list, &callbacks);

                while (!list_empty(&callbacks)) {
                        struct blk_plug_cb *cb = list_first_entry(&callbacks,
                                                          struct blk_plug_cb,
                                                          list);
                        list_del(&cb->list);
                        cb->callback(cb, from_schedule);
                }
        }
}

struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
                                      int size)
{
        struct blk_plug *plug = current->plug;
        struct blk_plug_cb *cb;

        if (!plug)
                return NULL;

        list_for_each_entry(cb, &plug->cb_list, list)
                if (cb->callback == unplug && cb->data == data)
                        return cb;

        /* Not currently on the callback list */
        BUG_ON(size < sizeof(*cb));
        cb = kzalloc(size, GFP_ATOMIC);
        if (cb) {
                cb->data = data;
                cb->callback = unplug;
                list_add(&cb->list, &plug->cb_list);
        }
        return cb;
}
EXPORT_SYMBOL(blk_check_plugged);

void __blk_flush_plug(struct blk_plug *plug, bool from_schedule)
{
        if (!list_empty(&plug->cb_list))
                flush_plug_callbacks(plug, from_schedule);
        blk_mq_flush_plug_list(plug, from_schedule);
        /*
         * Unconditionally flush out cached requests, even if the unplug
         * event came from schedule. Since we know hold references to the
         * queue for cached requests, we don't want a blocked task holding
         * up a queue freeze/quiesce event.
         */
        if (unlikely(!rq_list_empty(&plug->cached_rqs)))
                blk_mq_free_plug_rqs(plug);

        plug->cur_ktime = 0;
        current->flags &= ~PF_BLOCK_TS;
}

/**
 * blk_finish_plug - mark the end of a batch of submitted I/O
 * @plug:        The &struct blk_plug passed to blk_start_plug()
 *
 * Description:
 * Indicate that a batch of I/O submissions is complete.  This function
 * must be paired with an initial call to blk_start_plug().  The intent
 * is to allow the block layer to optimize I/O submission.  See the
 * documentation for blk_start_plug() for more information.
 */
void blk_finish_plug(struct blk_plug *plug)
{
        if (plug == current->plug) {
                __blk_flush_plug(plug, false);
                current->plug = NULL;
        }
}
EXPORT_SYMBOL(blk_finish_plug);

void blk_io_schedule(void)
{
        /* Prevent hang_check timer from firing at us during very long I/O */
        unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2;

        if (timeout)
                io_schedule_timeout(timeout);
        else
                io_schedule();
}
EXPORT_SYMBOL_GPL(blk_io_schedule);

int __init blk_dev_init(void)
{
        BUILD_BUG_ON((__force u32)REQ_OP_LAST >= (1 << REQ_OP_BITS));
        BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
                        sizeof_field(struct request, cmd_flags));
        BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
                        sizeof_field(struct bio, bi_opf));

        /* used for unplugging and affects IO latency/throughput - HIGHPRI */
        kblockd_workqueue = alloc_workqueue("kblockd",
                                            WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU, 0);
        if (!kblockd_workqueue)
                panic("Failed to create kblockd\n");

        blk_requestq_cachep = KMEM_CACHE(request_queue, SLAB_PANIC);

        blk_debugfs_root = debugfs_create_dir("block", NULL);

        return 0;
}













































































   52 
   53 


















































































































    9 
























































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
/*
 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#ifndef _CORE_PRIV_H
#define _CORE_PRIV_H

#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/cgroup_rdma.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>

#include <rdma/ib_verbs.h>
#include <rdma/opa_addr.h>
#include <rdma/ib_mad.h>
#include <rdma/restrack.h>
#include "mad_priv.h"
#include "restrack.h"

/* Total number of ports combined across all struct ib_devices's */
#define RDMA_MAX_PORTS 8192

struct pkey_index_qp_list {
        struct list_head    pkey_index_list;
        u16                 pkey_index;
        /* Lock to hold while iterating the qp_list. */
        spinlock_t          qp_list_lock;
        struct list_head    qp_list;
};

/**
 * struct rdma_dev_net - rdma net namespace metadata for a net
 * @nl_sock:        Pointer to netlink socket
 * @net:        Pointer to owner net namespace
 * @id:                xarray id to identify the net namespace.
 */
struct rdma_dev_net {
        struct sock *nl_sock;
        possible_net_t net;
        u32 id;
};

extern const struct attribute_group ib_dev_attr_group;
extern bool ib_devices_shared_netns;
extern unsigned int rdma_dev_net_id;

static inline struct rdma_dev_net *rdma_net_to_dev_net(struct net *net)
{
        return net_generic(net, rdma_dev_net_id);
}

int ib_device_rename(struct ib_device *ibdev, const char *name);
int ib_device_set_dim(struct ib_device *ibdev, u8 use_dim);

typedef void (*roce_netdev_callback)(struct ib_device *device, u32 port,
              struct net_device *idev, void *cookie);

typedef bool (*roce_netdev_filter)(struct ib_device *device, u32 port,
                                   struct net_device *idev, void *cookie);

struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,
                                        u32 port);

void ib_enum_roce_netdev(struct ib_device *ib_dev,
                         roce_netdev_filter filter,
                         void *filter_cookie,
                         roce_netdev_callback cb,
                         void *cookie);
void ib_enum_all_roce_netdevs(roce_netdev_filter filter,
                              void *filter_cookie,
                              roce_netdev_callback cb,
                              void *cookie);

void ib_device_enable_gid_updates(struct ib_device *device);
void ib_device_disable_gid_updates(struct ib_device *device);

typedef int (*nldev_callback)(struct ib_device *device,
                              struct sk_buff *skb,
                              struct netlink_callback *cb,
                              unsigned int idx);

int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
                     struct netlink_callback *cb);

struct ib_client_nl_info {
        struct sk_buff *nl_msg;
        struct device *cdev;
        u32 port;
        u64 abi;
};
int ib_get_client_nl_info(struct ib_device *ibdev, const char *client_name,
                          struct ib_client_nl_info *res);

enum ib_cache_gid_default_mode {
        IB_CACHE_GID_DEFAULT_MODE_SET,
        IB_CACHE_GID_DEFAULT_MODE_DELETE
};

int ib_cache_gid_parse_type_str(const char *buf);

const char *ib_cache_gid_type_str(enum ib_gid_type gid_type);

void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u32 port,
                                  struct net_device *ndev,
                                  unsigned long gid_type_mask,
                                  enum ib_cache_gid_default_mode mode);

int ib_cache_gid_add(struct ib_device *ib_dev, u32 port,
                     union ib_gid *gid, struct ib_gid_attr *attr);

int ib_cache_gid_del(struct ib_device *ib_dev, u32 port,
                     union ib_gid *gid, struct ib_gid_attr *attr);

int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u32 port,
                                     struct net_device *ndev);

int roce_gid_mgmt_init(void);
void roce_gid_mgmt_cleanup(void);

unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u32 port);

int ib_cache_setup_one(struct ib_device *device);
void ib_cache_cleanup_one(struct ib_device *device);
void ib_cache_release_one(struct ib_device *device);
void ib_dispatch_event_clients(struct ib_event *event);

#ifdef CONFIG_CGROUP_RDMA
void ib_device_register_rdmacg(struct ib_device *device);
void ib_device_unregister_rdmacg(struct ib_device *device);

int ib_rdmacg_try_charge(struct ib_rdmacg_object *cg_obj,
                         struct ib_device *device,
                         enum rdmacg_resource_type resource_index);

void ib_rdmacg_uncharge(struct ib_rdmacg_object *cg_obj,
                        struct ib_device *device,
                        enum rdmacg_resource_type resource_index);
#else
static inline void ib_device_register_rdmacg(struct ib_device *device)
{
}

static inline void ib_device_unregister_rdmacg(struct ib_device *device)
{
}

static inline int ib_rdmacg_try_charge(struct ib_rdmacg_object *cg_obj,
                                       struct ib_device *device,
                                       enum rdmacg_resource_type resource_index)
{
        return 0;
}

static inline void ib_rdmacg_uncharge(struct ib_rdmacg_object *cg_obj,
                                      struct ib_device *device,
                                      enum rdmacg_resource_type resource_index)
{
}
#endif

static inline bool rdma_is_upper_dev_rcu(struct net_device *dev,
                                         struct net_device *upper)
{
        return netdev_has_upper_dev_all_rcu(dev, upper);
}

int addr_init(void);
void addr_cleanup(void);

int ib_mad_init(void);
void ib_mad_cleanup(void);

int ib_sa_init(void);
void ib_sa_cleanup(void);

void rdma_nl_init(void);
void rdma_nl_exit(void);

int ib_nl_handle_resolve_resp(struct sk_buff *skb,
                              struct nlmsghdr *nlh,
                              struct netlink_ext_ack *extack);
int ib_nl_handle_set_timeout(struct sk_buff *skb,
                             struct nlmsghdr *nlh,
                             struct netlink_ext_ack *extack);
int ib_nl_handle_ip_res_resp(struct sk_buff *skb,
                             struct nlmsghdr *nlh,
                             struct netlink_ext_ack *extack);

void ib_get_cached_subnet_prefix(struct ib_device *device,
                                u32 port_num,
                                u64 *sn_pfx);

#ifdef CONFIG_SECURITY_INFINIBAND
void ib_security_release_port_pkey_list(struct ib_device *device);

void ib_security_cache_change(struct ib_device *device,
                              u32 port_num,
                              u64 subnet_prefix);

int ib_security_modify_qp(struct ib_qp *qp,
                          struct ib_qp_attr *qp_attr,
                          int qp_attr_mask,
                          struct ib_udata *udata);

int ib_create_qp_security(struct ib_qp *qp, struct ib_device *dev);
void ib_destroy_qp_security_begin(struct ib_qp_security *sec);
void ib_destroy_qp_security_abort(struct ib_qp_security *sec);
void ib_destroy_qp_security_end(struct ib_qp_security *sec);
int ib_open_shared_qp_security(struct ib_qp *qp, struct ib_device *dev);
void ib_close_shared_qp_security(struct ib_qp_security *sec);
int ib_mad_agent_security_setup(struct ib_mad_agent *agent,
                                enum ib_qp_type qp_type);
void ib_mad_agent_security_cleanup(struct ib_mad_agent *agent);
int ib_mad_enforce_security(struct ib_mad_agent_private *map, u16 pkey_index);
void ib_mad_agent_security_change(void);
#else
static inline void ib_security_release_port_pkey_list(struct ib_device *device)
{
}

static inline void ib_security_cache_change(struct ib_device *device,
                                            u32 port_num,
                                            u64 subnet_prefix)
{
}

static inline int ib_security_modify_qp(struct ib_qp *qp,
                                        struct ib_qp_attr *qp_attr,
                                        int qp_attr_mask,
                                        struct ib_udata *udata)
{
        return qp->device->ops.modify_qp(qp->real_qp,
                                         qp_attr,
                                         qp_attr_mask,
                                         udata);
}

static inline int ib_create_qp_security(struct ib_qp *qp,
                                        struct ib_device *dev)
{
        return 0;
}

static inline void ib_destroy_qp_security_begin(struct ib_qp_security *sec)
{
}

static inline void ib_destroy_qp_security_abort(struct ib_qp_security *sec)
{
}

static inline void ib_destroy_qp_security_end(struct ib_qp_security *sec)
{
}

static inline int ib_open_shared_qp_security(struct ib_qp *qp,
                                             struct ib_device *dev)
{
        return 0;
}

static inline void ib_close_shared_qp_security(struct ib_qp_security *sec)
{
}

static inline int ib_mad_agent_security_setup(struct ib_mad_agent *agent,
                                              enum ib_qp_type qp_type)
{
        return 0;
}

static inline void ib_mad_agent_security_cleanup(struct ib_mad_agent *agent)
{
}

static inline int ib_mad_enforce_security(struct ib_mad_agent_private *map,
                                          u16 pkey_index)
{
        return 0;
}

static inline void ib_mad_agent_security_change(void)
{
}
#endif

struct ib_device *ib_device_get_by_index(const struct net *net, u32 index);

/* RDMA device netlink */
void nldev_init(void);
void nldev_exit(void);

struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd,
                                struct ib_qp_init_attr *attr,
                                struct ib_udata *udata,
                                struct ib_uqp_object *uobj, const char *caller);

void ib_qp_usecnt_inc(struct ib_qp *qp);
void ib_qp_usecnt_dec(struct ib_qp *qp);

struct rdma_dev_addr;

int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
                                 const union ib_gid *dgid,
                                 u8 *dmac, const struct ib_gid_attr *sgid_attr,
                                 int *hoplimit);
void rdma_copy_src_l2_addr(struct rdma_dev_addr *dev_addr,
                           const struct net_device *dev);

struct sa_path_rec;
int roce_resolve_route_from_path(struct sa_path_rec *rec,
                                 const struct ib_gid_attr *attr);

struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr);

void ib_free_port_attrs(struct ib_core_device *coredev);
int ib_setup_port_attrs(struct ib_core_device *coredev);
struct rdma_hw_stats *ib_get_hw_stats_port(struct ib_device *ibdev, u32 port_num);
void ib_device_release_hw_stats(struct hw_stats_device_data *data);
int ib_setup_device_attrs(struct ib_device *ibdev);

int rdma_compatdev_set(u8 enable);

int ib_port_register_client_groups(struct ib_device *ibdev, u32 port_num,
                                   const struct attribute_group **groups);
void ib_port_unregister_client_groups(struct ib_device *ibdev, u32 port_num,
                                     const struct attribute_group **groups);

int ib_device_set_netns_put(struct sk_buff *skb,
                            struct ib_device *dev, u32 ns_fd);

int rdma_nl_net_init(struct rdma_dev_net *rnet);
void rdma_nl_net_exit(struct rdma_dev_net *rnet);

struct rdma_umap_priv {
        struct vm_area_struct *vma;
        struct list_head list;
        struct rdma_user_mmap_entry *entry;
};

void rdma_umap_priv_init(struct rdma_umap_priv *priv,
                         struct vm_area_struct *vma,
                         struct rdma_user_mmap_entry *entry);

void ib_cq_pool_cleanup(struct ib_device *dev);

bool rdma_nl_get_privileged_qkey(void);
#endif /* _CORE_PRIV_H */



































































  782 



































  786 





























  791 










  790 







   25 













  788 






  786 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
// SPDX-License-Identifier: GPL-2.0
/*
 * kmod - the kernel module loader
 *
 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
 */

#include <linux/module.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
#include <linux/binfmts.h>
#include <linux/syscalls.h>
#include <linux/unistd.h>
#include <linux/kmod.h>
#include <linux/slab.h>
#include <linux/completion.h>
#include <linux/cred.h>
#include <linux/file.h>
#include <linux/workqueue.h>
#include <linux/security.h>
#include <linux/mount.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/resource.h>
#include <linux/notifier.h>
#include <linux/suspend.h>
#include <linux/rwsem.h>
#include <linux/ptrace.h>
#include <linux/async.h>
#include <linux/uaccess.h>

#include <trace/events/module.h>
#include "internal.h"

/*
 * Assuming:
 *
 * threads = div64_u64((u64) totalram_pages * (u64) PAGE_SIZE,
 *                       (u64) THREAD_SIZE * 8UL);
 *
 * If you need less than 50 threads would mean we're dealing with systems
 * smaller than 3200 pages. This assumes you are capable of having ~13M memory,
 * and this would only be an upper limit, after which the OOM killer would take
 * effect. Systems like these are very unlikely if modules are enabled.
 */
#define MAX_KMOD_CONCURRENT 50
static DEFINE_SEMAPHORE(kmod_concurrent_max, MAX_KMOD_CONCURRENT);

/*
 * This is a restriction on having *all* MAX_KMOD_CONCURRENT threads
 * running at the same time without returning. When this happens we
 * believe you've somehow ended up with a recursive module dependency
 * creating a loop.
 *
 * We have no option but to fail.
 *
 * Userspace should proactively try to detect and prevent these.
 */
#define MAX_KMOD_ALL_BUSY_TIMEOUT 5

/*
        modprobe_path is set via /proc/sys.
*/
char modprobe_path[KMOD_PATH_LEN] = CONFIG_MODPROBE_PATH;

static void free_modprobe_argv(struct subprocess_info *info)
{
        kfree(info->argv[3]); /* check call_modprobe() */
        kfree(info->argv);
}

static int call_modprobe(char *orig_module_name, int wait)
{
        struct subprocess_info *info;
        static char *envp[] = {
                "HOME=/",
                "TERM=linux",
                "PATH=/sbin:/usr/sbin:/bin:/usr/bin",
                NULL
        };
        char *module_name;
        int ret;

        char **argv = kmalloc(sizeof(char *[5]), GFP_KERNEL);
        if (!argv)
                goto out;

        module_name = kstrdup(orig_module_name, GFP_KERNEL);
        if (!module_name)
                goto free_argv;

        argv[0] = modprobe_path;
        argv[1] = "-q";
        argv[2] = "--";
        argv[3] = module_name;        /* check free_modprobe_argv() */
        argv[4] = NULL;

        info = call_usermodehelper_setup(modprobe_path, argv, envp, GFP_KERNEL,
                                         NULL, free_modprobe_argv, NULL);
        if (!info)
                goto free_module_name;

        ret = call_usermodehelper_exec(info, wait | UMH_KILLABLE);
        kmod_dup_request_announce(orig_module_name, ret);
        return ret;

free_module_name:
        kfree(module_name);
free_argv:
        kfree(argv);
out:
        kmod_dup_request_announce(orig_module_name, -ENOMEM);
        return -ENOMEM;
}

/**
 * __request_module - try to load a kernel module
 * @wait: wait (or not) for the operation to complete
 * @fmt: printf style format string for the name of the module
 * @...: arguments as specified in the format string
 *
 * Load a module using the user mode module loader. The function returns
 * zero on success or a negative errno code or positive exit code from
 * "modprobe" on failure. Note that a successful module load does not mean
 * the module did not then unload and exit on an error of its own. Callers
 * must check that the service they requested is now available not blindly
 * invoke it.
 *
 * If module auto-loading support is disabled then this function
 * simply returns -ENOENT.
 */
int __request_module(bool wait, const char *fmt, ...)
{
        va_list args;
        char module_name[MODULE_NAME_LEN];
        int ret, dup_ret;

        /*
         * We don't allow synchronous module loading from async.  Module
         * init may invoke async_synchronize_full() which will end up
         * waiting for this task which already is waiting for the module
         * loading to complete, leading to a deadlock.
         */
        WARN_ON_ONCE(wait && current_is_async());

        if (!modprobe_path[0])
                return -ENOENT;

        va_start(args, fmt);
        ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
        va_end(args);
        if (ret >= MODULE_NAME_LEN)
                return -ENAMETOOLONG;

        ret = security_kernel_module_request(module_name);
        if (ret)
                return ret;

        ret = down_timeout(&kmod_concurrent_max, MAX_KMOD_ALL_BUSY_TIMEOUT * HZ);
        if (ret) {
                pr_warn_ratelimited("request_module: modprobe %s cannot be processed, kmod busy with %d threads for more than %d seconds now",
                                    module_name, MAX_KMOD_CONCURRENT, MAX_KMOD_ALL_BUSY_TIMEOUT);
                return ret;
        }

        trace_module_request(module_name, wait, _RET_IP_);

        if (kmod_dup_request_exists_wait(module_name, wait, &dup_ret)) {
                ret = dup_ret;
                goto out;
        }

        ret = call_modprobe(module_name, wait ? UMH_WAIT_PROC : UMH_WAIT_EXEC);

out:
        up(&kmod_concurrent_max);

        return ret;
}
EXPORT_SYMBOL(__request_module);


























































   25 


   26 








































   58 
  372 

  230 


  408 




  377 
  299 

  387 












































































  242 

    4 



  227 

    9 





    5 





    5 






























    4 





    4 












































    3 
    2 






   21 







   17 












































    5 

















































































































    8 
  227 








  436 





  428 




    5 







    4 




    2 





    3 
    1 



  419 




    2 
   10 



  410 












   53 










    2 
    1 




    1 
    1 








    9 






















    9 
















   13 



    2 






    2 
    2 









    3 




    1 
    4 





    8 




    8 














    1 
    1 




   11 





   11 







   86 







   68 




    7 






    1 
    1 



    2 

    2 
    2 



    4 




    8 




    7 




    4 




    6 



    4 




    3 


    2 


    2 



    5 



    2 



    2 



   14 








    2 



   21 










    2 



   13 










   10 







   52 




   52 



    6 











   24 


    7 



    1 
    1 
    1 
    1 

    6 

    9 


    1 
    4 



    6 
    3 



    2 
    4 






























































































































































  365 








    4 

































  229 


































  363 




  365 


  229 

























  229 





    4 



  361 




















    2 
    3 



  238 







  409 


  406 


    2 



  405 


    4 



  403 
    6 










    2 




    5 


  402 


  400 
    5 

    6 


  321 

   85 

    1 


  318 




    1 




    3 


  318 



    3 


  318 



    3 


  316 




    5 



  318 




    2 
    2 
    2 
    2 
    2 


  319 




    1 


    1 


    1 
    1 
    1 











  321 

   85 
  240 

































































   22 











   17 




    1 










    1 

































    1 




    6 




    1 





    2 





    1 
    6 














    1 
    4 



    5 

    1 













  361 



   90 






   20 

   20 
   20 






    1 




    2 




   17 

  236 
   14 



  232 



    5 

    2 


    3 

    3 

    3 

    3 

    3 

    3 




  236 





































  234 


    1 

    3 





  253 























  246 

    7 

    5 


  231 








    1 





   13 
   13 


   13 





  241 



    3 
    3 



  241 
    3 





  241 



    3 
    3 



  241 



    4 
    4 



  242 



    3 

    3 




  242 



    2 

    2 




  245 














  244 
    1 









  224 




  226 

    5 

  226 

  226 
    2 




























  383 







   18 


  365 




  117 


   12 
    6 





















  226 







    2 



  229 
  230 







  230 



  242 











  274 




    1 
    5 











  291 
  280 
    4 
    8 






  189 


    1 






  188 






  413 
  413 

























   48 































































































































































































































































































































































































   57 













   18 




































  243 








  202 


  110 
  108 





   57 



    7 



   18 














   55 
    3 


  244 

















  110 







  110 








    7 












   67 






   74 


    3 
   63 







    7 


   74 










   74 




































   13 

    5 



    2 



    5 



    2 



















































   28 



    4 



    2 




   21 


    2 
    5 




    5 
    5 



   18 



   18 






   16 


    2 















    1 


   57 



   57 









   52 


    5 








  382 




  296 
   86 

   54 
  329 



































    1 











    1 









































































    6 
    3 














   10 
   10 









   35 





    6 







    6 



   23 


    9 














    1 






























































    2 







    3 







    3 






    2 











    4 


















    3 























    3 


    4 




    6 
























    3 


    3 







    2 

































































    3 
    4 


    2 
    1 




   48 
















  243 







  110 
































  172 
   67 




    9 




   13 





   26 



























    4 





    2 


    1 












    7 




    3 







    2 













    1 





    2 












    2 

































   16 







   19 






   16 











   15 

    7 








    1 





    2 







   13 


    8 




   19 






    2 


















   58 












    8 












   29 









    4 




   10 








   17 
   48 















  244 







  244 






   52 
  192 





































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2007-2017 Nicira, Inc.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include "flow.h"
#include "datapath.h"
#include <linux/uaccess.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <net/llc_pdu.h>
#include <linux/kernel.h>
#include <linux/jhash.h>
#include <linux/jiffies.h>
#include <linux/llc.h>
#include <linux/module.h>
#include <linux/in.h>
#include <linux/rcupdate.h>
#include <linux/if_arp.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/sctp.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/icmp.h>
#include <linux/icmpv6.h>
#include <linux/rculist.h>
#include <net/geneve.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/ndisc.h>
#include <net/mpls.h>
#include <net/vxlan.h>
#include <net/tun_proto.h>
#include <net/erspan.h>

#include "drop.h"
#include "flow_netlink.h"

struct ovs_len_tbl {
        int len;
        const struct ovs_len_tbl *next;
};

#define OVS_ATTR_NESTED -1
#define OVS_ATTR_VARIABLE -2
#define OVS_COPY_ACTIONS_MAX_DEPTH 16

static bool actions_may_change_flow(const struct nlattr *actions)
{
        struct nlattr *nla;
        int rem;

        nla_for_each_nested(nla, actions, rem) {
                u16 action = nla_type(nla);

                switch (action) {
                case OVS_ACTION_ATTR_OUTPUT:
                case OVS_ACTION_ATTR_RECIRC:
                case OVS_ACTION_ATTR_TRUNC:
                case OVS_ACTION_ATTR_USERSPACE:
                case OVS_ACTION_ATTR_DROP:
                case OVS_ACTION_ATTR_PSAMPLE:
                        break;

                case OVS_ACTION_ATTR_CT:
                case OVS_ACTION_ATTR_CT_CLEAR:
                case OVS_ACTION_ATTR_HASH:
                case OVS_ACTION_ATTR_POP_ETH:
                case OVS_ACTION_ATTR_POP_MPLS:
                case OVS_ACTION_ATTR_POP_NSH:
                case OVS_ACTION_ATTR_POP_VLAN:
                case OVS_ACTION_ATTR_PUSH_ETH:
                case OVS_ACTION_ATTR_PUSH_MPLS:
                case OVS_ACTION_ATTR_PUSH_NSH:
                case OVS_ACTION_ATTR_PUSH_VLAN:
                case OVS_ACTION_ATTR_SAMPLE:
                case OVS_ACTION_ATTR_SET:
                case OVS_ACTION_ATTR_SET_MASKED:
                case OVS_ACTION_ATTR_METER:
                case OVS_ACTION_ATTR_CHECK_PKT_LEN:
                case OVS_ACTION_ATTR_ADD_MPLS:
                case OVS_ACTION_ATTR_DEC_TTL:
                default:
                        return true;
                }
        }
        return false;
}

static void update_range(struct sw_flow_match *match,
                         size_t offset, size_t size, bool is_mask)
{
        struct sw_flow_key_range *range;
        size_t start = rounddown(offset, sizeof(long));
        size_t end = roundup(offset + size, sizeof(long));

        if (!is_mask)
                range = &match->range;
        else
                range = &match->mask->range;

        if (range->start == range->end) {
                range->start = start;
                range->end = end;
                return;
        }

        if (range->start > start)
                range->start = start;

        if (range->end < end)
                range->end = end;
}

#define SW_FLOW_KEY_PUT(match, field, value, is_mask) \
        do { \
                update_range(match, offsetof(struct sw_flow_key, field),    \
                             sizeof((match)->key->field), is_mask);            \
                if (is_mask)                                                    \
                        (match)->mask->key.field = value;                    \
                else                                                            \
                        (match)->key->field = value;                            \
        } while (0)

#define SW_FLOW_KEY_MEMCPY_OFFSET(match, offset, value_p, len, is_mask)            \
        do {                                                                    \
                update_range(match, offset, len, is_mask);                    \
                if (is_mask)                                                    \
                        memcpy((u8 *)&(match)->mask->key + offset, value_p, \
                               len);                                           \
                else                                                            \
                        memcpy((u8 *)(match)->key + offset, value_p, len);  \
        } while (0)

#define SW_FLOW_KEY_MEMCPY(match, field, value_p, len, is_mask)                      \
        SW_FLOW_KEY_MEMCPY_OFFSET(match, offsetof(struct sw_flow_key, field), \
                                  value_p, len, is_mask)

#define SW_FLOW_KEY_MEMSET_FIELD(match, field, value, is_mask)                    \
        do {                                                                    \
                update_range(match, offsetof(struct sw_flow_key, field),    \
                             sizeof((match)->key->field), is_mask);            \
                if (is_mask)                                                    \
                        memset((u8 *)&(match)->mask->key.field, value,      \
                               sizeof((match)->mask->key.field));            \
                else                                                            \
                        memset((u8 *)&(match)->key->field, value,           \
                               sizeof((match)->key->field));                \
        } while (0)

#define SW_FLOW_KEY_BITMAP_COPY(match, field, value_p, nbits, is_mask) ({     \
        update_range(match, offsetof(struct sw_flow_key, field),              \
                     bitmap_size(nbits), is_mask);                              \
        bitmap_copy(is_mask ? (match)->mask->key.field : (match)->key->field, \
                    value_p, nbits);                                              \
})

static bool match_validate(const struct sw_flow_match *match,
                           u64 key_attrs, u64 mask_attrs, bool log)
{
        u64 key_expected = 0;
        u64 mask_allowed = key_attrs;  /* At most allow all key attributes */

        /* The following mask attributes allowed only if they
         * pass the validation tests. */
        mask_allowed &= ~((1 << OVS_KEY_ATTR_IPV4)
                        | (1 << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4)
                        | (1 << OVS_KEY_ATTR_IPV6)
                        | (1 << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6)
                        | (1 << OVS_KEY_ATTR_TCP)
                        | (1 << OVS_KEY_ATTR_TCP_FLAGS)
                        | (1 << OVS_KEY_ATTR_UDP)
                        | (1 << OVS_KEY_ATTR_SCTP)
                        | (1 << OVS_KEY_ATTR_ICMP)
                        | (1 << OVS_KEY_ATTR_ICMPV6)
                        | (1 << OVS_KEY_ATTR_ARP)
                        | (1 << OVS_KEY_ATTR_ND)
                        | (1 << OVS_KEY_ATTR_MPLS)
                        | (1 << OVS_KEY_ATTR_NSH));

        /* Always allowed mask fields. */
        mask_allowed |= ((1 << OVS_KEY_ATTR_TUNNEL)
                       | (1 << OVS_KEY_ATTR_IN_PORT)
                       | (1 << OVS_KEY_ATTR_ETHERTYPE));

        /* Check key attributes. */
        if (match->key->eth.type == htons(ETH_P_ARP)
                        || match->key->eth.type == htons(ETH_P_RARP)) {
                key_expected |= 1 << OVS_KEY_ATTR_ARP;
                if (match->mask && (match->mask->key.eth.type == htons(0xffff)))
                        mask_allowed |= 1 << OVS_KEY_ATTR_ARP;
        }

        if (eth_p_mpls(match->key->eth.type)) {
                key_expected |= 1 << OVS_KEY_ATTR_MPLS;
                if (match->mask && (match->mask->key.eth.type == htons(0xffff)))
                        mask_allowed |= 1 << OVS_KEY_ATTR_MPLS;
        }

        if (match->key->eth.type == htons(ETH_P_IP)) {
                key_expected |= 1 << OVS_KEY_ATTR_IPV4;
                if (match->mask && match->mask->key.eth.type == htons(0xffff)) {
                        mask_allowed |= 1 << OVS_KEY_ATTR_IPV4;
                        mask_allowed |= 1 << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4;
                }

                if (match->key->ip.frag != OVS_FRAG_TYPE_LATER) {
                        if (match->key->ip.proto == IPPROTO_UDP) {
                                key_expected |= 1 << OVS_KEY_ATTR_UDP;
                                if (match->mask && (match->mask->key.ip.proto == 0xff))
                                        mask_allowed |= 1 << OVS_KEY_ATTR_UDP;
                        }

                        if (match->key->ip.proto == IPPROTO_SCTP) {
                                key_expected |= 1 << OVS_KEY_ATTR_SCTP;
                                if (match->mask && (match->mask->key.ip.proto == 0xff))
                                        mask_allowed |= 1 << OVS_KEY_ATTR_SCTP;
                        }

                        if (match->key->ip.proto == IPPROTO_TCP) {
                                key_expected |= 1 << OVS_KEY_ATTR_TCP;
                                key_expected |= 1 << OVS_KEY_ATTR_TCP_FLAGS;
                                if (match->mask && (match->mask->key.ip.proto == 0xff)) {
                                        mask_allowed |= 1 << OVS_KEY_ATTR_TCP;
                                        mask_allowed |= 1 << OVS_KEY_ATTR_TCP_FLAGS;
                                }
                        }

                        if (match->key->ip.proto == IPPROTO_ICMP) {
                                key_expected |= 1 << OVS_KEY_ATTR_ICMP;
                                if (match->mask && (match->mask->key.ip.proto == 0xff))
                                        mask_allowed |= 1 << OVS_KEY_ATTR_ICMP;
                        }
                }
        }

        if (match->key->eth.type == htons(ETH_P_IPV6)) {
                key_expected |= 1 << OVS_KEY_ATTR_IPV6;
                if (match->mask && match->mask->key.eth.type == htons(0xffff)) {
                        mask_allowed |= 1 << OVS_KEY_ATTR_IPV6;
                        mask_allowed |= 1 << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6;
                }

                if (match->key->ip.frag != OVS_FRAG_TYPE_LATER) {
                        if (match->key->ip.proto == IPPROTO_UDP) {
                                key_expected |= 1 << OVS_KEY_ATTR_UDP;
                                if (match->mask && (match->mask->key.ip.proto == 0xff))
                                        mask_allowed |= 1 << OVS_KEY_ATTR_UDP;
                        }

                        if (match->key->ip.proto == IPPROTO_SCTP) {
                                key_expected |= 1 << OVS_KEY_ATTR_SCTP;
                                if (match->mask && (match->mask->key.ip.proto == 0xff))
                                        mask_allowed |= 1 << OVS_KEY_ATTR_SCTP;
                        }

                        if (match->key->ip.proto == IPPROTO_TCP) {
                                key_expected |= 1 << OVS_KEY_ATTR_TCP;
                                key_expected |= 1 << OVS_KEY_ATTR_TCP_FLAGS;
                                if (match->mask && (match->mask->key.ip.proto == 0xff)) {
                                        mask_allowed |= 1 << OVS_KEY_ATTR_TCP;
                                        mask_allowed |= 1 << OVS_KEY_ATTR_TCP_FLAGS;
                                }
                        }

                        if (match->key->ip.proto == IPPROTO_ICMPV6) {
                                key_expected |= 1 << OVS_KEY_ATTR_ICMPV6;
                                if (match->mask && (match->mask->key.ip.proto == 0xff))
                                        mask_allowed |= 1 << OVS_KEY_ATTR_ICMPV6;

                                if (match->key->tp.src ==
                                                htons(NDISC_NEIGHBOUR_SOLICITATION) ||
                                    match->key->tp.src == htons(NDISC_NEIGHBOUR_ADVERTISEMENT)) {
                                        key_expected |= 1 << OVS_KEY_ATTR_ND;
                                        /* Original direction conntrack tuple
                                         * uses the same space as the ND fields
                                         * in the key, so both are not allowed
                                         * at the same time.
                                         */
                                        mask_allowed &= ~(1ULL << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6);
                                        if (match->mask && (match->mask->key.tp.src == htons(0xff)))
                                                mask_allowed |= 1 << OVS_KEY_ATTR_ND;
                                }
                        }
                }
        }

        if (match->key->eth.type == htons(ETH_P_NSH)) {
                key_expected |= 1 << OVS_KEY_ATTR_NSH;
                if (match->mask &&
                    match->mask->key.eth.type == htons(0xffff)) {
                        mask_allowed |= 1 << OVS_KEY_ATTR_NSH;
                }
        }

        if ((key_attrs & key_expected) != key_expected) {
                /* Key attributes check failed. */
                OVS_NLERR(log, "Missing key (keys=%llx, expected=%llx)",
                          (unsigned long long)key_attrs,
                          (unsigned long long)key_expected);
                return false;
        }

        if ((mask_attrs & mask_allowed) != mask_attrs) {
                /* Mask attributes check failed. */
                OVS_NLERR(log, "Unexpected mask (mask=%llx, allowed=%llx)",
                          (unsigned long long)mask_attrs,
                          (unsigned long long)mask_allowed);
                return false;
        }

        return true;
}

size_t ovs_tun_key_attr_size(void)
{
        /* Whenever adding new OVS_TUNNEL_KEY_ FIELDS, we should consider
         * updating this function.
         */
        return    nla_total_size_64bit(8) /* OVS_TUNNEL_KEY_ATTR_ID */
                + nla_total_size(16)   /* OVS_TUNNEL_KEY_ATTR_IPV[46]_SRC */
                + nla_total_size(16)   /* OVS_TUNNEL_KEY_ATTR_IPV[46]_DST */
                + nla_total_size(1)    /* OVS_TUNNEL_KEY_ATTR_TOS */
                + nla_total_size(1)    /* OVS_TUNNEL_KEY_ATTR_TTL */
                + nla_total_size(0)    /* OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT */
                + nla_total_size(0)    /* OVS_TUNNEL_KEY_ATTR_CSUM */
                + nla_total_size(0)    /* OVS_TUNNEL_KEY_ATTR_OAM */
                + nla_total_size(256)  /* OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS */
                /* OVS_TUNNEL_KEY_ATTR_VXLAN_OPTS and
                 * OVS_TUNNEL_KEY_ATTR_ERSPAN_OPTS is mutually exclusive with
                 * OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS and covered by it.
                 */
                + nla_total_size(2)    /* OVS_TUNNEL_KEY_ATTR_TP_SRC */
                + nla_total_size(2);   /* OVS_TUNNEL_KEY_ATTR_TP_DST */
}

static size_t ovs_nsh_key_attr_size(void)
{
        /* Whenever adding new OVS_NSH_KEY_ FIELDS, we should consider
         * updating this function.
         */
        return  nla_total_size(NSH_BASE_HDR_LEN) /* OVS_NSH_KEY_ATTR_BASE */
                /* OVS_NSH_KEY_ATTR_MD1 and OVS_NSH_KEY_ATTR_MD2 are
                 * mutually exclusive, so the bigger one can cover
                 * the small one.
                 */
                + nla_total_size(NSH_CTX_HDRS_MAX_LEN);
}

size_t ovs_key_attr_size(void)
{
        /* Whenever adding new OVS_KEY_ FIELDS, we should consider
         * updating this function.
         */
        BUILD_BUG_ON(OVS_KEY_ATTR_MAX != 32);

        return    nla_total_size(4)   /* OVS_KEY_ATTR_PRIORITY */
                + nla_total_size(0)   /* OVS_KEY_ATTR_TUNNEL */
                  + ovs_tun_key_attr_size()
                + nla_total_size(4)   /* OVS_KEY_ATTR_IN_PORT */
                + nla_total_size(4)   /* OVS_KEY_ATTR_SKB_MARK */
                + nla_total_size(4)   /* OVS_KEY_ATTR_DP_HASH */
                + nla_total_size(4)   /* OVS_KEY_ATTR_RECIRC_ID */
                + nla_total_size(4)   /* OVS_KEY_ATTR_CT_STATE */
                + nla_total_size(2)   /* OVS_KEY_ATTR_CT_ZONE */
                + nla_total_size(4)   /* OVS_KEY_ATTR_CT_MARK */
                + nla_total_size(16)  /* OVS_KEY_ATTR_CT_LABELS */
                + nla_total_size(40)  /* OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6 */
                + nla_total_size(0)   /* OVS_KEY_ATTR_NSH */
                  + ovs_nsh_key_attr_size()
                + nla_total_size(12)  /* OVS_KEY_ATTR_ETHERNET */
                + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
                + nla_total_size(4)   /* OVS_KEY_ATTR_VLAN */
                + nla_total_size(0)   /* OVS_KEY_ATTR_ENCAP */
                + nla_total_size(2)   /* OVS_KEY_ATTR_ETHERTYPE */
                + nla_total_size(40)  /* OVS_KEY_ATTR_IPV6 */
                + nla_total_size(2)   /* OVS_KEY_ATTR_ICMPV6 */
                + nla_total_size(28)  /* OVS_KEY_ATTR_ND */
                + nla_total_size(2);  /* OVS_KEY_ATTR_IPV6_EXTHDRS */
}

static const struct ovs_len_tbl ovs_vxlan_ext_key_lens[OVS_VXLAN_EXT_MAX + 1] = {
        [OVS_VXLAN_EXT_GBP]            = { .len = sizeof(u32) },
};

static const struct ovs_len_tbl ovs_tunnel_key_lens[OVS_TUNNEL_KEY_ATTR_MAX + 1] = {
        [OVS_TUNNEL_KEY_ATTR_ID]            = { .len = sizeof(u64) },
        [OVS_TUNNEL_KEY_ATTR_IPV4_SRC]            = { .len = sizeof(u32) },
        [OVS_TUNNEL_KEY_ATTR_IPV4_DST]            = { .len = sizeof(u32) },
        [OVS_TUNNEL_KEY_ATTR_TOS]            = { .len = 1 },
        [OVS_TUNNEL_KEY_ATTR_TTL]            = { .len = 1 },
        [OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT] = { .len = 0 },
        [OVS_TUNNEL_KEY_ATTR_CSUM]            = { .len = 0 },
        [OVS_TUNNEL_KEY_ATTR_TP_SRC]            = { .len = sizeof(u16) },
        [OVS_TUNNEL_KEY_ATTR_TP_DST]            = { .len = sizeof(u16) },
        [OVS_TUNNEL_KEY_ATTR_OAM]            = { .len = 0 },
        [OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS]   = { .len = OVS_ATTR_VARIABLE },
        [OVS_TUNNEL_KEY_ATTR_VXLAN_OPTS]    = { .len = OVS_ATTR_NESTED,
                                                .next = ovs_vxlan_ext_key_lens },
        [OVS_TUNNEL_KEY_ATTR_IPV6_SRC]      = { .len = sizeof(struct in6_addr) },
        [OVS_TUNNEL_KEY_ATTR_IPV6_DST]      = { .len = sizeof(struct in6_addr) },
        [OVS_TUNNEL_KEY_ATTR_ERSPAN_OPTS]   = { .len = OVS_ATTR_VARIABLE },
        [OVS_TUNNEL_KEY_ATTR_IPV4_INFO_BRIDGE]   = { .len = 0 },
};

static const struct ovs_len_tbl
ovs_nsh_key_attr_lens[OVS_NSH_KEY_ATTR_MAX + 1] = {
        [OVS_NSH_KEY_ATTR_BASE] = { .len = sizeof(struct ovs_nsh_key_base) },
        [OVS_NSH_KEY_ATTR_MD1]  = { .len = sizeof(struct ovs_nsh_key_md1) },
        [OVS_NSH_KEY_ATTR_MD2]  = { .len = OVS_ATTR_VARIABLE },
};

/* The size of the argument for each %OVS_KEY_ATTR_* Netlink attribute.  */
static const struct ovs_len_tbl ovs_key_lens[OVS_KEY_ATTR_MAX + 1] = {
        [OVS_KEY_ATTR_ENCAP]         = { .len = OVS_ATTR_NESTED },
        [OVS_KEY_ATTR_PRIORITY]         = { .len = sizeof(u32) },
        [OVS_KEY_ATTR_IN_PORT]         = { .len = sizeof(u32) },
        [OVS_KEY_ATTR_SKB_MARK]         = { .len = sizeof(u32) },
        [OVS_KEY_ATTR_ETHERNET]         = { .len = sizeof(struct ovs_key_ethernet) },
        [OVS_KEY_ATTR_VLAN]         = { .len = sizeof(__be16) },
        [OVS_KEY_ATTR_ETHERTYPE] = { .len = sizeof(__be16) },
        [OVS_KEY_ATTR_IPV4]         = { .len = sizeof(struct ovs_key_ipv4) },
        [OVS_KEY_ATTR_IPV6]         = { .len = sizeof(struct ovs_key_ipv6) },
        [OVS_KEY_ATTR_TCP]         = { .len = sizeof(struct ovs_key_tcp) },
        [OVS_KEY_ATTR_TCP_FLAGS] = { .len = sizeof(__be16) },
        [OVS_KEY_ATTR_UDP]         = { .len = sizeof(struct ovs_key_udp) },
        [OVS_KEY_ATTR_SCTP]         = { .len = sizeof(struct ovs_key_sctp) },
        [OVS_KEY_ATTR_ICMP]         = { .len = sizeof(struct ovs_key_icmp) },
        [OVS_KEY_ATTR_ICMPV6]         = { .len = sizeof(struct ovs_key_icmpv6) },
        [OVS_KEY_ATTR_ARP]         = { .len = sizeof(struct ovs_key_arp) },
        [OVS_KEY_ATTR_ND]         = { .len = sizeof(struct ovs_key_nd) },
        [OVS_KEY_ATTR_RECIRC_ID] = { .len = sizeof(u32) },
        [OVS_KEY_ATTR_DP_HASH]         = { .len = sizeof(u32) },
        [OVS_KEY_ATTR_TUNNEL]         = { .len = OVS_ATTR_NESTED,
                                     .next = ovs_tunnel_key_lens, },
        [OVS_KEY_ATTR_MPLS]         = { .len = OVS_ATTR_VARIABLE },
        [OVS_KEY_ATTR_CT_STATE]         = { .len = sizeof(u32) },
        [OVS_KEY_ATTR_CT_ZONE]         = { .len = sizeof(u16) },
        [OVS_KEY_ATTR_CT_MARK]         = { .len = sizeof(u32) },
        [OVS_KEY_ATTR_CT_LABELS] = { .len = sizeof(struct ovs_key_ct_labels) },
        [OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4] = {
                .len = sizeof(struct ovs_key_ct_tuple_ipv4) },
        [OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6] = {
                .len = sizeof(struct ovs_key_ct_tuple_ipv6) },
        [OVS_KEY_ATTR_NSH]       = { .len = OVS_ATTR_NESTED,
                                     .next = ovs_nsh_key_attr_lens, },
        [OVS_KEY_ATTR_IPV6_EXTHDRS] = {
                .len = sizeof(struct ovs_key_ipv6_exthdrs) },
};

static bool check_attr_len(unsigned int attr_len, unsigned int expected_len)
{
        return expected_len == attr_len ||
               expected_len == OVS_ATTR_NESTED ||
               expected_len == OVS_ATTR_VARIABLE;
}

static bool is_all_zero(const u8 *fp, size_t size)
{
        int i;

        if (!fp)
                return false;

        for (i = 0; i < size; i++)
                if (fp[i])
                        return false;

        return true;
}

static int __parse_flow_nlattrs(const struct nlattr *attr,
                                const struct nlattr *a[],
                                u64 *attrsp, bool log, bool nz)
{
        const struct nlattr *nla;
        u64 attrs;
        int rem;

        attrs = *attrsp;
        nla_for_each_nested(nla, attr, rem) {
                u16 type = nla_type(nla);
                int expected_len;

                if (type > OVS_KEY_ATTR_MAX) {
                        OVS_NLERR(log, "Key type %d is out of range max %d",
                                  type, OVS_KEY_ATTR_MAX);
                        return -EINVAL;
                }

                if (type == OVS_KEY_ATTR_PACKET_TYPE ||
                    type == OVS_KEY_ATTR_ND_EXTENSIONS ||
                    type == OVS_KEY_ATTR_TUNNEL_INFO) {
                        OVS_NLERR(log, "Key type %d is not supported", type);
                        return -EINVAL;
                }

                if (attrs & (1ULL << type)) {
                        OVS_NLERR(log, "Duplicate key (type %d).", type);
                        return -EINVAL;
                }

                expected_len = ovs_key_lens[type].len;
                if (!check_attr_len(nla_len(nla), expected_len)) {
                        OVS_NLERR(log, "Key %d has unexpected len %d expected %d",
                                  type, nla_len(nla), expected_len);
                        return -EINVAL;
                }

                if (!nz || !is_all_zero(nla_data(nla), nla_len(nla))) {
                        attrs |= 1ULL << type;
                        a[type] = nla;
                }
        }
        if (rem) {
                OVS_NLERR(log, "Message has %d unknown bytes.", rem);
                return -EINVAL;
        }

        *attrsp = attrs;
        return 0;
}

static int parse_flow_mask_nlattrs(const struct nlattr *attr,
                                   const struct nlattr *a[], u64 *attrsp,
                                   bool log)
{
        return __parse_flow_nlattrs(attr, a, attrsp, log, true);
}

int parse_flow_nlattrs(const struct nlattr *attr, const struct nlattr *a[],
                       u64 *attrsp, bool log)
{
        return __parse_flow_nlattrs(attr, a, attrsp, log, false);
}

static int genev_tun_opt_from_nlattr(const struct nlattr *a,
                                     struct sw_flow_match *match, bool is_mask,
                                     bool log)
{
        unsigned long opt_key_offset;

        if (nla_len(a) > sizeof(match->key->tun_opts)) {
                OVS_NLERR(log, "Geneve option length err (len %d, max %zu).",
                          nla_len(a), sizeof(match->key->tun_opts));
                return -EINVAL;
        }

        if (nla_len(a) % 4 != 0) {
                OVS_NLERR(log, "Geneve opt len %d is not a multiple of 4.",
                          nla_len(a));
                return -EINVAL;
        }

        /* We need to record the length of the options passed
         * down, otherwise packets with the same format but
         * additional options will be silently matched.
         */
        if (!is_mask) {
                SW_FLOW_KEY_PUT(match, tun_opts_len, nla_len(a),
                                false);
        } else {
                /* This is somewhat unusual because it looks at
                 * both the key and mask while parsing the
                 * attributes (and by extension assumes the key
                 * is parsed first). Normally, we would verify
                 * that each is the correct length and that the
                 * attributes line up in the validate function.
                 * However, that is difficult because this is
                 * variable length and we won't have the
                 * information later.
                 */
                if (match->key->tun_opts_len != nla_len(a)) {
                        OVS_NLERR(log, "Geneve option len %d != mask len %d",
                                  match->key->tun_opts_len, nla_len(a));
                        return -EINVAL;
                }

                SW_FLOW_KEY_PUT(match, tun_opts_len, 0xff, true);
        }

        opt_key_offset = TUN_METADATA_OFFSET(nla_len(a));
        SW_FLOW_KEY_MEMCPY_OFFSET(match, opt_key_offset, nla_data(a),
                                  nla_len(a), is_mask);
        return 0;
}

static int vxlan_tun_opt_from_nlattr(const struct nlattr *attr,
                                     struct sw_flow_match *match, bool is_mask,
                                     bool log)
{
        struct nlattr *a;
        int rem;
        unsigned long opt_key_offset;
        struct vxlan_metadata opts;

        BUILD_BUG_ON(sizeof(opts) > sizeof(match->key->tun_opts));

        memset(&opts, 0, sizeof(opts));
        nla_for_each_nested(a, attr, rem) {
                int type = nla_type(a);

                if (type > OVS_VXLAN_EXT_MAX) {
                        OVS_NLERR(log, "VXLAN extension %d out of range max %d",
                                  type, OVS_VXLAN_EXT_MAX);
                        return -EINVAL;
                }

                if (!check_attr_len(nla_len(a),
                                    ovs_vxlan_ext_key_lens[type].len)) {
                        OVS_NLERR(log, "VXLAN extension %d has unexpected len %d expected %d",
                                  type, nla_len(a),
                                  ovs_vxlan_ext_key_lens[type].len);
                        return -EINVAL;
                }

                switch (type) {
                case OVS_VXLAN_EXT_GBP:
                        opts.gbp = nla_get_u32(a);
                        break;
                default:
                        OVS_NLERR(log, "Unknown VXLAN extension attribute %d",
                                  type);
                        return -EINVAL;
                }
        }
        if (rem) {
                OVS_NLERR(log, "VXLAN extension message has %d unknown bytes.",
                          rem);
                return -EINVAL;
        }

        if (!is_mask)
                SW_FLOW_KEY_PUT(match, tun_opts_len, sizeof(opts), false);
        else
                SW_FLOW_KEY_PUT(match, tun_opts_len, 0xff, true);

        opt_key_offset = TUN_METADATA_OFFSET(sizeof(opts));
        SW_FLOW_KEY_MEMCPY_OFFSET(match, opt_key_offset, &opts, sizeof(opts),
                                  is_mask);
        return 0;
}

static int erspan_tun_opt_from_nlattr(const struct nlattr *a,
                                      struct sw_flow_match *match, bool is_mask,
                                      bool log)
{
        unsigned long opt_key_offset;

        BUILD_BUG_ON(sizeof(struct erspan_metadata) >
                     sizeof(match->key->tun_opts));

        if (nla_len(a) > sizeof(match->key->tun_opts)) {
                OVS_NLERR(log, "ERSPAN option length err (len %d, max %zu).",
                          nla_len(a), sizeof(match->key->tun_opts));
                return -EINVAL;
        }

        if (!is_mask)
                SW_FLOW_KEY_PUT(match, tun_opts_len,
                                sizeof(struct erspan_metadata), false);
        else
                SW_FLOW_KEY_PUT(match, tun_opts_len, 0xff, true);

        opt_key_offset = TUN_METADATA_OFFSET(nla_len(a));
        SW_FLOW_KEY_MEMCPY_OFFSET(match, opt_key_offset, nla_data(a),
                                  nla_len(a), is_mask);
        return 0;
}

static int ip_tun_from_nlattr(const struct nlattr *attr,
                              struct sw_flow_match *match, bool is_mask,
                              bool log)
{
        bool ttl = false, ipv4 = false, ipv6 = false;
        IP_TUNNEL_DECLARE_FLAGS(tun_flags) = { };
        bool info_bridge_mode = false;
        int opts_type = 0;
        struct nlattr *a;
        int rem;

        nla_for_each_nested(a, attr, rem) {
                int type = nla_type(a);
                int err;

                if (type > OVS_TUNNEL_KEY_ATTR_MAX) {
                        OVS_NLERR(log, "Tunnel attr %d out of range max %d",
                                  type, OVS_TUNNEL_KEY_ATTR_MAX);
                        return -EINVAL;
                }

                if (!check_attr_len(nla_len(a),
                                    ovs_tunnel_key_lens[type].len)) {
                        OVS_NLERR(log, "Tunnel attr %d has unexpected len %d expected %d",
                                  type, nla_len(a), ovs_tunnel_key_lens[type].len);
                        return -EINVAL;
                }

                switch (type) {
                case OVS_TUNNEL_KEY_ATTR_ID:
                        SW_FLOW_KEY_PUT(match, tun_key.tun_id,
                                        nla_get_be64(a), is_mask);
                        __set_bit(IP_TUNNEL_KEY_BIT, tun_flags);
                        break;
                case OVS_TUNNEL_KEY_ATTR_IPV4_SRC:
                        SW_FLOW_KEY_PUT(match, tun_key.u.ipv4.src,
                                        nla_get_in_addr(a), is_mask);
                        ipv4 = true;
                        break;
                case OVS_TUNNEL_KEY_ATTR_IPV4_DST:
                        SW_FLOW_KEY_PUT(match, tun_key.u.ipv4.dst,
                                        nla_get_in_addr(a), is_mask);
                        ipv4 = true;
                        break;
                case OVS_TUNNEL_KEY_ATTR_IPV6_SRC:
                        SW_FLOW_KEY_PUT(match, tun_key.u.ipv6.src,
                                        nla_get_in6_addr(a), is_mask);
                        ipv6 = true;
                        break;
                case OVS_TUNNEL_KEY_ATTR_IPV6_DST:
                        SW_FLOW_KEY_PUT(match, tun_key.u.ipv6.dst,
                                        nla_get_in6_addr(a), is_mask);
                        ipv6 = true;
                        break;
                case OVS_TUNNEL_KEY_ATTR_TOS:
                        SW_FLOW_KEY_PUT(match, tun_key.tos,
                                        nla_get_u8(a), is_mask);
                        break;
                case OVS_TUNNEL_KEY_ATTR_TTL:
                        SW_FLOW_KEY_PUT(match, tun_key.ttl,
                                        nla_get_u8(a), is_mask);
                        ttl = true;
                        break;
                case OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT:
                        __set_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, tun_flags);
                        break;
                case OVS_TUNNEL_KEY_ATTR_CSUM:
                        __set_bit(IP_TUNNEL_CSUM_BIT, tun_flags);
                        break;
                case OVS_TUNNEL_KEY_ATTR_TP_SRC:
                        SW_FLOW_KEY_PUT(match, tun_key.tp_src,
                                        nla_get_be16(a), is_mask);
                        break;
                case OVS_TUNNEL_KEY_ATTR_TP_DST:
                        SW_FLOW_KEY_PUT(match, tun_key.tp_dst,
                                        nla_get_be16(a), is_mask);
                        break;
                case OVS_TUNNEL_KEY_ATTR_OAM:
                        __set_bit(IP_TUNNEL_OAM_BIT, tun_flags);
                        break;
                case OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS:
                        if (opts_type) {
                                OVS_NLERR(log, "Multiple metadata blocks provided");
                                return -EINVAL;
                        }

                        err = genev_tun_opt_from_nlattr(a, match, is_mask, log);
                        if (err)
                                return err;

                        __set_bit(IP_TUNNEL_GENEVE_OPT_BIT, tun_flags);
                        opts_type = type;
                        break;
                case OVS_TUNNEL_KEY_ATTR_VXLAN_OPTS:
                        if (opts_type) {
                                OVS_NLERR(log, "Multiple metadata blocks provided");
                                return -EINVAL;
                        }

                        err = vxlan_tun_opt_from_nlattr(a, match, is_mask, log);
                        if (err)
                                return err;

                        __set_bit(IP_TUNNEL_VXLAN_OPT_BIT, tun_flags);
                        opts_type = type;
                        break;
                case OVS_TUNNEL_KEY_ATTR_PAD:
                        break;
                case OVS_TUNNEL_KEY_ATTR_ERSPAN_OPTS:
                        if (opts_type) {
                                OVS_NLERR(log, "Multiple metadata blocks provided");
                                return -EINVAL;
                        }

                        err = erspan_tun_opt_from_nlattr(a, match, is_mask,
                                                         log);
                        if (err)
                                return err;

                        __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, tun_flags);
                        opts_type = type;
                        break;
                case OVS_TUNNEL_KEY_ATTR_IPV4_INFO_BRIDGE:
                        info_bridge_mode = true;
                        ipv4 = true;
                        break;
                default:
                        OVS_NLERR(log, "Unknown IP tunnel attribute %d",
                                  type);
                        return -EINVAL;
                }
        }

        SW_FLOW_KEY_BITMAP_COPY(match, tun_key.tun_flags, tun_flags,
                                __IP_TUNNEL_FLAG_NUM, is_mask);
        if (is_mask)
                SW_FLOW_KEY_MEMSET_FIELD(match, tun_proto, 0xff, true);
        else
                SW_FLOW_KEY_PUT(match, tun_proto, ipv6 ? AF_INET6 : AF_INET,
                                false);

        if (rem > 0) {
                OVS_NLERR(log, "IP tunnel attribute has %d unknown bytes.",
                          rem);
                return -EINVAL;
        }

        if (ipv4 && ipv6) {
                OVS_NLERR(log, "Mixed IPv4 and IPv6 tunnel attributes");
                return -EINVAL;
        }

        if (!is_mask) {
                if (!ipv4 && !ipv6) {
                        OVS_NLERR(log, "IP tunnel dst address not specified");
                        return -EINVAL;
                }
                if (ipv4) {
                        if (info_bridge_mode) {
                                __clear_bit(IP_TUNNEL_KEY_BIT, tun_flags);

                                if (match->key->tun_key.u.ipv4.src ||
                                    match->key->tun_key.u.ipv4.dst ||
                                    match->key->tun_key.tp_src ||
                                    match->key->tun_key.tp_dst ||
                                    match->key->tun_key.ttl ||
                                    match->key->tun_key.tos ||
                                    !ip_tunnel_flags_empty(tun_flags)) {
                                        OVS_NLERR(log, "IPv4 tun info is not correct");
                                        return -EINVAL;
                                }
                        } else if (!match->key->tun_key.u.ipv4.dst) {
                                OVS_NLERR(log, "IPv4 tunnel dst address is zero");
                                return -EINVAL;
                        }
                }
                if (ipv6 && ipv6_addr_any(&match->key->tun_key.u.ipv6.dst)) {
                        OVS_NLERR(log, "IPv6 tunnel dst address is zero");
                        return -EINVAL;
                }

                if (!ttl && !info_bridge_mode) {
                        OVS_NLERR(log, "IP tunnel TTL not specified.");
                        return -EINVAL;
                }
        }

        return opts_type;
}

static int vxlan_opt_to_nlattr(struct sk_buff *skb,
                               const void *tun_opts, int swkey_tun_opts_len)
{
        const struct vxlan_metadata *opts = tun_opts;
        struct nlattr *nla;

        nla = nla_nest_start_noflag(skb, OVS_TUNNEL_KEY_ATTR_VXLAN_OPTS);
        if (!nla)
                return -EMSGSIZE;

        if (nla_put_u32(skb, OVS_VXLAN_EXT_GBP, opts->gbp) < 0)
                return -EMSGSIZE;

        nla_nest_end(skb, nla);
        return 0;
}

static int __ip_tun_to_nlattr(struct sk_buff *skb,
                              const struct ip_tunnel_key *output,
                              const void *tun_opts, int swkey_tun_opts_len,
                              unsigned short tun_proto, u8 mode)
{
        if (test_bit(IP_TUNNEL_KEY_BIT, output->tun_flags) &&
            nla_put_be64(skb, OVS_TUNNEL_KEY_ATTR_ID, output->tun_id,
                         OVS_TUNNEL_KEY_ATTR_PAD))
                return -EMSGSIZE;

        if (mode & IP_TUNNEL_INFO_BRIDGE)
                return nla_put_flag(skb, OVS_TUNNEL_KEY_ATTR_IPV4_INFO_BRIDGE)
                       ? -EMSGSIZE : 0;

        switch (tun_proto) {
        case AF_INET:
                if (output->u.ipv4.src &&
                    nla_put_in_addr(skb, OVS_TUNNEL_KEY_ATTR_IPV4_SRC,
                                    output->u.ipv4.src))
                        return -EMSGSIZE;
                if (output->u.ipv4.dst &&
                    nla_put_in_addr(skb, OVS_TUNNEL_KEY_ATTR_IPV4_DST,
                                    output->u.ipv4.dst))
                        return -EMSGSIZE;
                break;
        case AF_INET6:
                if (!ipv6_addr_any(&output->u.ipv6.src) &&
                    nla_put_in6_addr(skb, OVS_TUNNEL_KEY_ATTR_IPV6_SRC,
                                     &output->u.ipv6.src))
                        return -EMSGSIZE;
                if (!ipv6_addr_any(&output->u.ipv6.dst) &&
                    nla_put_in6_addr(skb, OVS_TUNNEL_KEY_ATTR_IPV6_DST,
                                     &output->u.ipv6.dst))
                        return -EMSGSIZE;
                break;
        }
        if (output->tos &&
            nla_put_u8(skb, OVS_TUNNEL_KEY_ATTR_TOS, output->tos))
                return -EMSGSIZE;
        if (nla_put_u8(skb, OVS_TUNNEL_KEY_ATTR_TTL, output->ttl))
                return -EMSGSIZE;
        if (test_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, output->tun_flags) &&
            nla_put_flag(skb, OVS_TUNNEL_KEY_ATTR_DONT_FRAGMENT))
                return -EMSGSIZE;
        if (test_bit(IP_TUNNEL_CSUM_BIT, output->tun_flags) &&
            nla_put_flag(skb, OVS_TUNNEL_KEY_ATTR_CSUM))
                return -EMSGSIZE;
        if (output->tp_src &&
            nla_put_be16(skb, OVS_TUNNEL_KEY_ATTR_TP_SRC, output->tp_src))
                return -EMSGSIZE;
        if (output->tp_dst &&
            nla_put_be16(skb, OVS_TUNNEL_KEY_ATTR_TP_DST, output->tp_dst))
                return -EMSGSIZE;
        if (test_bit(IP_TUNNEL_OAM_BIT, output->tun_flags) &&
            nla_put_flag(skb, OVS_TUNNEL_KEY_ATTR_OAM))
                return -EMSGSIZE;
        if (swkey_tun_opts_len) {
                if (test_bit(IP_TUNNEL_GENEVE_OPT_BIT, output->tun_flags) &&
                    nla_put(skb, OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS,
                            swkey_tun_opts_len, tun_opts))
                        return -EMSGSIZE;
                else if (test_bit(IP_TUNNEL_VXLAN_OPT_BIT,
                                  output->tun_flags) &&
                         vxlan_opt_to_nlattr(skb, tun_opts, swkey_tun_opts_len))
                        return -EMSGSIZE;
                else if (test_bit(IP_TUNNEL_ERSPAN_OPT_BIT,
                                  output->tun_flags) &&
                         nla_put(skb, OVS_TUNNEL_KEY_ATTR_ERSPAN_OPTS,
                                 swkey_tun_opts_len, tun_opts))
                        return -EMSGSIZE;
        }

        return 0;
}

static int ip_tun_to_nlattr(struct sk_buff *skb,
                            const struct ip_tunnel_key *output,
                            const void *tun_opts, int swkey_tun_opts_len,
                            unsigned short tun_proto, u8 mode)
{
        struct nlattr *nla;
        int err;

        nla = nla_nest_start_noflag(skb, OVS_KEY_ATTR_TUNNEL);
        if (!nla)
                return -EMSGSIZE;

        err = __ip_tun_to_nlattr(skb, output, tun_opts, swkey_tun_opts_len,
                                 tun_proto, mode);
        if (err)
                return err;

        nla_nest_end(skb, nla);
        return 0;
}

int ovs_nla_put_tunnel_info(struct sk_buff *skb,
                            struct ip_tunnel_info *tun_info)
{
        return __ip_tun_to_nlattr(skb, &tun_info->key,
                                  ip_tunnel_info_opts(tun_info),
                                  tun_info->options_len,
                                  ip_tunnel_info_af(tun_info), tun_info->mode);
}

static int encode_vlan_from_nlattrs(struct sw_flow_match *match,
                                    const struct nlattr *a[],
                                    bool is_mask, bool inner)
{
        __be16 tci = 0;
        __be16 tpid = 0;

        if (a[OVS_KEY_ATTR_VLAN])
                tci = nla_get_be16(a[OVS_KEY_ATTR_VLAN]);

        if (a[OVS_KEY_ATTR_ETHERTYPE])
                tpid = nla_get_be16(a[OVS_KEY_ATTR_ETHERTYPE]);

        if (likely(!inner)) {
                SW_FLOW_KEY_PUT(match, eth.vlan.tpid, tpid, is_mask);
                SW_FLOW_KEY_PUT(match, eth.vlan.tci, tci, is_mask);
        } else {
                SW_FLOW_KEY_PUT(match, eth.cvlan.tpid, tpid, is_mask);
                SW_FLOW_KEY_PUT(match, eth.cvlan.tci, tci, is_mask);
        }
        return 0;
}

static int validate_vlan_from_nlattrs(const struct sw_flow_match *match,
                                      u64 key_attrs, bool inner,
                                      const struct nlattr **a, bool log)
{
        __be16 tci = 0;

        if (!((key_attrs & (1 << OVS_KEY_ATTR_ETHERNET)) &&
              (key_attrs & (1 << OVS_KEY_ATTR_ETHERTYPE)) &&
               eth_type_vlan(nla_get_be16(a[OVS_KEY_ATTR_ETHERTYPE])))) {
                /* Not a VLAN. */
                return 0;
        }

        if (!((key_attrs & (1 << OVS_KEY_ATTR_VLAN)) &&
              (key_attrs & (1 << OVS_KEY_ATTR_ENCAP)))) {
                OVS_NLERR(log, "Invalid %s frame", (inner) ? "C-VLAN" : "VLAN");
                return -EINVAL;
        }

        if (a[OVS_KEY_ATTR_VLAN])
                tci = nla_get_be16(a[OVS_KEY_ATTR_VLAN]);

        if (!(tci & htons(VLAN_CFI_MASK))) {
                if (tci) {
                        OVS_NLERR(log, "%s TCI does not have VLAN_CFI_MASK bit set.",
                                  (inner) ? "C-VLAN" : "VLAN");
                        return -EINVAL;
                } else if (nla_len(a[OVS_KEY_ATTR_ENCAP])) {
                        /* Corner case for truncated VLAN header. */
                        OVS_NLERR(log, "Truncated %s header has non-zero encap attribute.",
                                  (inner) ? "C-VLAN" : "VLAN");
                        return -EINVAL;
                }
        }

        return 1;
}

static int validate_vlan_mask_from_nlattrs(const struct sw_flow_match *match,
                                           u64 key_attrs, bool inner,
                                           const struct nlattr **a, bool log)
{
        __be16 tci = 0;
        __be16 tpid = 0;
        bool encap_valid = !!(match->key->eth.vlan.tci &
                              htons(VLAN_CFI_MASK));
        bool i_encap_valid = !!(match->key->eth.cvlan.tci &
                                htons(VLAN_CFI_MASK));

        if (!(key_attrs & (1 << OVS_KEY_ATTR_ENCAP))) {
                /* Not a VLAN. */
                return 0;
        }

        if ((!inner && !encap_valid) || (inner && !i_encap_valid)) {
                OVS_NLERR(log, "Encap mask attribute is set for non-%s frame.",
                          (inner) ? "C-VLAN" : "VLAN");
                return -EINVAL;
        }

        if (a[OVS_KEY_ATTR_VLAN])
                tci = nla_get_be16(a[OVS_KEY_ATTR_VLAN]);

        if (a[OVS_KEY_ATTR_ETHERTYPE])
                tpid = nla_get_be16(a[OVS_KEY_ATTR_ETHERTYPE]);

        if (tpid != htons(0xffff)) {
                OVS_NLERR(log, "Must have an exact match on %s TPID (mask=%x).",
                          (inner) ? "C-VLAN" : "VLAN", ntohs(tpid));
                return -EINVAL;
        }
        if (!(tci & htons(VLAN_CFI_MASK))) {
                OVS_NLERR(log, "%s TCI mask does not have exact match for VLAN_CFI_MASK bit.",
                          (inner) ? "C-VLAN" : "VLAN");
                return -EINVAL;
        }

        return 1;
}

static int __parse_vlan_from_nlattrs(struct sw_flow_match *match,
                                     u64 *key_attrs, bool inner,
                                     const struct nlattr **a, bool is_mask,
                                     bool log)
{
        int err;
        const struct nlattr *encap;

        if (!is_mask)
                err = validate_vlan_from_nlattrs(match, *key_attrs, inner,
                                                 a, log);
        else
                err = validate_vlan_mask_from_nlattrs(match, *key_attrs, inner,
                                                      a, log);
        if (err <= 0)
                return err;

        err = encode_vlan_from_nlattrs(match, a, is_mask, inner);
        if (err)
                return err;

        *key_attrs &= ~(1 << OVS_KEY_ATTR_ENCAP);
        *key_attrs &= ~(1 << OVS_KEY_ATTR_VLAN);
        *key_attrs &= ~(1 << OVS_KEY_ATTR_ETHERTYPE);

        encap = a[OVS_KEY_ATTR_ENCAP];

        if (!is_mask)
                err = parse_flow_nlattrs(encap, a, key_attrs, log);
        else
                err = parse_flow_mask_nlattrs(encap, a, key_attrs, log);

        return err;
}

static int parse_vlan_from_nlattrs(struct sw_flow_match *match,
                                   u64 *key_attrs, const struct nlattr **a,
                                   bool is_mask, bool log)
{
        int err;
        bool encap_valid = false;

        err = __parse_vlan_from_nlattrs(match, key_attrs, false, a,
                                        is_mask, log);
        if (err)
                return err;

        encap_valid = !!(match->key->eth.vlan.tci & htons(VLAN_CFI_MASK));
        if (encap_valid) {
                err = __parse_vlan_from_nlattrs(match, key_attrs, true, a,
                                                is_mask, log);
                if (err)
                        return err;
        }

        return 0;
}

static int parse_eth_type_from_nlattrs(struct sw_flow_match *match,
                                       u64 *attrs, const struct nlattr **a,
                                       bool is_mask, bool log)
{
        __be16 eth_type;

        eth_type = nla_get_be16(a[OVS_KEY_ATTR_ETHERTYPE]);
        if (is_mask) {
                /* Always exact match EtherType. */
                eth_type = htons(0xffff);
        } else if (!eth_proto_is_802_3(eth_type)) {
                OVS_NLERR(log, "EtherType %x is less than min %x",
                                ntohs(eth_type), ETH_P_802_3_MIN);
                return -EINVAL;
        }

        SW_FLOW_KEY_PUT(match, eth.type, eth_type, is_mask);
        *attrs &= ~(1 << OVS_KEY_ATTR_ETHERTYPE);
        return 0;
}

static int metadata_from_nlattrs(struct net *net, struct sw_flow_match *match,
                                 u64 *attrs, const struct nlattr **a,
                                 bool is_mask, bool log)
{
        u8 mac_proto = MAC_PROTO_ETHERNET;

        if (*attrs & (1 << OVS_KEY_ATTR_DP_HASH)) {
                u32 hash_val = nla_get_u32(a[OVS_KEY_ATTR_DP_HASH]);

                SW_FLOW_KEY_PUT(match, ovs_flow_hash, hash_val, is_mask);
                *attrs &= ~(1 << OVS_KEY_ATTR_DP_HASH);
        }

        if (*attrs & (1 << OVS_KEY_ATTR_RECIRC_ID)) {
                u32 recirc_id = nla_get_u32(a[OVS_KEY_ATTR_RECIRC_ID]);

                SW_FLOW_KEY_PUT(match, recirc_id, recirc_id, is_mask);
                *attrs &= ~(1 << OVS_KEY_ATTR_RECIRC_ID);
        }

        if (*attrs & (1 << OVS_KEY_ATTR_PRIORITY)) {
                SW_FLOW_KEY_PUT(match, phy.priority,
                          nla_get_u32(a[OVS_KEY_ATTR_PRIORITY]), is_mask);
                *attrs &= ~(1 << OVS_KEY_ATTR_PRIORITY);
        }

        if (*attrs & (1 << OVS_KEY_ATTR_IN_PORT)) {
                u32 in_port = nla_get_u32(a[OVS_KEY_ATTR_IN_PORT]);

                if (is_mask) {
                        in_port = 0xffffffff; /* Always exact match in_port. */
                } else if (in_port >= DP_MAX_PORTS) {
                        OVS_NLERR(log, "Port %d exceeds max allowable %d",
                                  in_port, DP_MAX_PORTS);
                        return -EINVAL;
                }

                SW_FLOW_KEY_PUT(match, phy.in_port, in_port, is_mask);
                *attrs &= ~(1 << OVS_KEY_ATTR_IN_PORT);
        } else if (!is_mask) {
                SW_FLOW_KEY_PUT(match, phy.in_port, DP_MAX_PORTS, is_mask);
        }

        if (*attrs & (1 << OVS_KEY_ATTR_SKB_MARK)) {
                uint32_t mark = nla_get_u32(a[OVS_KEY_ATTR_SKB_MARK]);

                SW_FLOW_KEY_PUT(match, phy.skb_mark, mark, is_mask);
                *attrs &= ~(1 << OVS_KEY_ATTR_SKB_MARK);
        }
        if (*attrs & (1 << OVS_KEY_ATTR_TUNNEL)) {
                if (ip_tun_from_nlattr(a[OVS_KEY_ATTR_TUNNEL], match,
                                       is_mask, log) < 0)
                        return -EINVAL;
                *attrs &= ~(1 << OVS_KEY_ATTR_TUNNEL);
        }

        if (*attrs & (1 << OVS_KEY_ATTR_CT_STATE) &&
            ovs_ct_verify(net, OVS_KEY_ATTR_CT_STATE)) {
                u32 ct_state = nla_get_u32(a[OVS_KEY_ATTR_CT_STATE]);

                if (ct_state & ~CT_SUPPORTED_MASK) {
                        OVS_NLERR(log, "ct_state flags %08x unsupported",
                                  ct_state);
                        return -EINVAL;
                }

                SW_FLOW_KEY_PUT(match, ct_state, ct_state, is_mask);
                *attrs &= ~(1ULL << OVS_KEY_ATTR_CT_STATE);
        }
        if (*attrs & (1 << OVS_KEY_ATTR_CT_ZONE) &&
            ovs_ct_verify(net, OVS_KEY_ATTR_CT_ZONE)) {
                u16 ct_zone = nla_get_u16(a[OVS_KEY_ATTR_CT_ZONE]);

                SW_FLOW_KEY_PUT(match, ct_zone, ct_zone, is_mask);
                *attrs &= ~(1ULL << OVS_KEY_ATTR_CT_ZONE);
        }
        if (*attrs & (1 << OVS_KEY_ATTR_CT_MARK) &&
            ovs_ct_verify(net, OVS_KEY_ATTR_CT_MARK)) {
                u32 mark = nla_get_u32(a[OVS_KEY_ATTR_CT_MARK]);

                SW_FLOW_KEY_PUT(match, ct.mark, mark, is_mask);
                *attrs &= ~(1ULL << OVS_KEY_ATTR_CT_MARK);
        }
        if (*attrs & (1 << OVS_KEY_ATTR_CT_LABELS) &&
            ovs_ct_verify(net, OVS_KEY_ATTR_CT_LABELS)) {
                const struct ovs_key_ct_labels *cl;

                cl = nla_data(a[OVS_KEY_ATTR_CT_LABELS]);
                SW_FLOW_KEY_MEMCPY(match, ct.labels, cl->ct_labels,
                                   sizeof(*cl), is_mask);
                *attrs &= ~(1ULL << OVS_KEY_ATTR_CT_LABELS);
        }
        if (*attrs & (1ULL << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4)) {
                const struct ovs_key_ct_tuple_ipv4 *ct;

                ct = nla_data(a[OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4]);

                SW_FLOW_KEY_PUT(match, ipv4.ct_orig.src, ct->ipv4_src, is_mask);
                SW_FLOW_KEY_PUT(match, ipv4.ct_orig.dst, ct->ipv4_dst, is_mask);
                SW_FLOW_KEY_PUT(match, ct.orig_tp.src, ct->src_port, is_mask);
                SW_FLOW_KEY_PUT(match, ct.orig_tp.dst, ct->dst_port, is_mask);
                SW_FLOW_KEY_PUT(match, ct_orig_proto, ct->ipv4_proto, is_mask);
                *attrs &= ~(1ULL << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4);
        }
        if (*attrs & (1ULL << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6)) {
                const struct ovs_key_ct_tuple_ipv6 *ct;

                ct = nla_data(a[OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6]);

                SW_FLOW_KEY_MEMCPY(match, ipv6.ct_orig.src, &ct->ipv6_src,
                                   sizeof(match->key->ipv6.ct_orig.src),
                                   is_mask);
                SW_FLOW_KEY_MEMCPY(match, ipv6.ct_orig.dst, &ct->ipv6_dst,
                                   sizeof(match->key->ipv6.ct_orig.dst),
                                   is_mask);
                SW_FLOW_KEY_PUT(match, ct.orig_tp.src, ct->src_port, is_mask);
                SW_FLOW_KEY_PUT(match, ct.orig_tp.dst, ct->dst_port, is_mask);
                SW_FLOW_KEY_PUT(match, ct_orig_proto, ct->ipv6_proto, is_mask);
                *attrs &= ~(1ULL << OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6);
        }

        /* For layer 3 packets the Ethernet type is provided
         * and treated as metadata but no MAC addresses are provided.
         */
        if (!(*attrs & (1ULL << OVS_KEY_ATTR_ETHERNET)) &&
            (*attrs & (1ULL << OVS_KEY_ATTR_ETHERTYPE)))
                mac_proto = MAC_PROTO_NONE;

        /* Always exact match mac_proto */
        SW_FLOW_KEY_PUT(match, mac_proto, is_mask ? 0xff : mac_proto, is_mask);

        if (mac_proto == MAC_PROTO_NONE)
                return parse_eth_type_from_nlattrs(match, attrs, a, is_mask,
                                                   log);

        return 0;
}

/*
 * Constructs NSH header 'nh' from attributes of OVS_ACTION_ATTR_PUSH_NSH,
 * where 'nh' points to a memory block of 'size' bytes.  It's assumed that
 * attributes were previously validated with validate_push_nsh().
 */
int nsh_hdr_from_nlattr(const struct nlattr *attr,
                        struct nshhdr *nh, size_t size)
{
        struct nlattr *a;
        int rem;
        u8 flags = 0;
        u8 ttl = 0;
        int mdlen = 0;

        if (size < NSH_BASE_HDR_LEN)
                return -ENOBUFS;

        nla_for_each_nested(a, attr, rem) {
                int type = nla_type(a);

                switch (type) {
                case OVS_NSH_KEY_ATTR_BASE: {
                        const struct ovs_nsh_key_base *base = nla_data(a);

                        flags = base->flags;
                        ttl = base->ttl;
                        nh->np = base->np;
                        nh->mdtype = base->mdtype;
                        nh->path_hdr = base->path_hdr;
                        break;
                }
                case OVS_NSH_KEY_ATTR_MD1:
                        mdlen = nla_len(a);
                        if (mdlen > size - NSH_BASE_HDR_LEN)
                                return -ENOBUFS;
                        memcpy(&nh->md1, nla_data(a), mdlen);
                        break;

                case OVS_NSH_KEY_ATTR_MD2:
                        mdlen = nla_len(a);
                        if (mdlen > size - NSH_BASE_HDR_LEN)
                                return -ENOBUFS;
                        memcpy(&nh->md2, nla_data(a), mdlen);
                        break;

                default:
                        return -EINVAL;
                }
        }

        /* nsh header length  = NSH_BASE_HDR_LEN + mdlen */
        nh->ver_flags_ttl_len = 0;
        nsh_set_flags_ttl_len(nh, flags, ttl, NSH_BASE_HDR_LEN + mdlen);

        return 0;
}

static int nsh_key_put_from_nlattr(const struct nlattr *attr,
                                   struct sw_flow_match *match, bool is_mask,
                                   bool is_push_nsh, bool log)
{
        struct nlattr *a;
        int rem;
        bool has_base = false;
        bool has_md1 = false;
        bool has_md2 = false;
        u8 mdtype = 0;
        int mdlen = 0;

        if (WARN_ON(is_push_nsh && is_mask))
                return -EINVAL;

        nla_for_each_nested(a, attr, rem) {
                int type = nla_type(a);
                int i;

                if (type > OVS_NSH_KEY_ATTR_MAX) {
                        OVS_NLERR(log, "nsh attr %d is out of range max %d",
                                  type, OVS_NSH_KEY_ATTR_MAX);
                        return -EINVAL;
                }

                if (!check_attr_len(nla_len(a),
                                    ovs_nsh_key_attr_lens[type].len)) {
                        OVS_NLERR(
                            log,
                            "nsh attr %d has unexpected len %d expected %d",
                            type,
                            nla_len(a),
                            ovs_nsh_key_attr_lens[type].len
                        );
                        return -EINVAL;
                }

                switch (type) {
                case OVS_NSH_KEY_ATTR_BASE: {
                        const struct ovs_nsh_key_base *base = nla_data(a);

                        has_base = true;
                        mdtype = base->mdtype;
                        SW_FLOW_KEY_PUT(match, nsh.base.flags,
                                        base->flags, is_mask);
                        SW_FLOW_KEY_PUT(match, nsh.base.ttl,
                                        base->ttl, is_mask);
                        SW_FLOW_KEY_PUT(match, nsh.base.mdtype,
                                        base->mdtype, is_mask);
                        SW_FLOW_KEY_PUT(match, nsh.base.np,
                                        base->np, is_mask);
                        SW_FLOW_KEY_PUT(match, nsh.base.path_hdr,
                                        base->path_hdr, is_mask);
                        break;
                }
                case OVS_NSH_KEY_ATTR_MD1: {
                        const struct ovs_nsh_key_md1 *md1 = nla_data(a);

                        has_md1 = true;
                        for (i = 0; i < NSH_MD1_CONTEXT_SIZE; i++)
                                SW_FLOW_KEY_PUT(match, nsh.context[i],
                                                md1->context[i], is_mask);
                        break;
                }
                case OVS_NSH_KEY_ATTR_MD2:
                        if (!is_push_nsh) /* Not supported MD type 2 yet */
                                return -ENOTSUPP;

                        has_md2 = true;
                        mdlen = nla_len(a);
                        if (mdlen > NSH_CTX_HDRS_MAX_LEN || mdlen <= 0) {
                                OVS_NLERR(
                                    log,
                                    "Invalid MD length %d for MD type %d",
                                    mdlen,
                                    mdtype
                                );
                                return -EINVAL;
                        }
                        break;
                default:
                        OVS_NLERR(log, "Unknown nsh attribute %d",
                                  type);
                        return -EINVAL;
                }
        }

        if (rem > 0) {
                OVS_NLERR(log, "nsh attribute has %d unknown bytes.", rem);
                return -EINVAL;
        }

        if (has_md1 && has_md2) {
                OVS_NLERR(
                    1,
                    "invalid nsh attribute: md1 and md2 are exclusive."
                );
                return -EINVAL;
        }

        if (!is_mask) {
                if ((has_md1 && mdtype != NSH_M_TYPE1) ||
                    (has_md2 && mdtype != NSH_M_TYPE2)) {
                        OVS_NLERR(1, "nsh attribute has unmatched MD type %d.",
                                  mdtype);
                        return -EINVAL;
                }

                if (is_push_nsh &&
                    (!has_base || (!has_md1 && !has_md2))) {
                        OVS_NLERR(
                            1,
                            "push_nsh: missing base or metadata attributes"
                        );
                        return -EINVAL;
                }
        }

        return 0;
}

static int ovs_key_from_nlattrs(struct net *net, struct sw_flow_match *match,
                                u64 attrs, const struct nlattr **a,
                                bool is_mask, bool log)
{
        int err;

        err = metadata_from_nlattrs(net, match, &attrs, a, is_mask, log);
        if (err)
                return err;

        if (attrs & (1 << OVS_KEY_ATTR_ETHERNET)) {
                const struct ovs_key_ethernet *eth_key;

                eth_key = nla_data(a[OVS_KEY_ATTR_ETHERNET]);
                SW_FLOW_KEY_MEMCPY(match, eth.src,
                                eth_key->eth_src, ETH_ALEN, is_mask);
                SW_FLOW_KEY_MEMCPY(match, eth.dst,
                                eth_key->eth_dst, ETH_ALEN, is_mask);
                attrs &= ~(1 << OVS_KEY_ATTR_ETHERNET);

                if (attrs & (1 << OVS_KEY_ATTR_VLAN)) {
                        /* VLAN attribute is always parsed before getting here since it
                         * may occur multiple times.
                         */
                        OVS_NLERR(log, "VLAN attribute unexpected.");
                        return -EINVAL;
                }

                if (attrs & (1 << OVS_KEY_ATTR_ETHERTYPE)) {
                        err = parse_eth_type_from_nlattrs(match, &attrs, a, is_mask,
                                                          log);
                        if (err)
                                return err;
                } else if (!is_mask) {
                        SW_FLOW_KEY_PUT(match, eth.type, htons(ETH_P_802_2), is_mask);
                }
        } else if (!match->key->eth.type) {
                OVS_NLERR(log, "Either Ethernet header or EtherType is required.");
                return -EINVAL;
        }

        if (attrs & (1 << OVS_KEY_ATTR_IPV4)) {
                const struct ovs_key_ipv4 *ipv4_key;

                ipv4_key = nla_data(a[OVS_KEY_ATTR_IPV4]);
                if (!is_mask && ipv4_key->ipv4_frag > OVS_FRAG_TYPE_MAX) {
                        OVS_NLERR(log, "IPv4 frag type %d is out of range max %d",
                                  ipv4_key->ipv4_frag, OVS_FRAG_TYPE_MAX);
                        return -EINVAL;
                }
                SW_FLOW_KEY_PUT(match, ip.proto,
                                ipv4_key->ipv4_proto, is_mask);
                SW_FLOW_KEY_PUT(match, ip.tos,
                                ipv4_key->ipv4_tos, is_mask);
                SW_FLOW_KEY_PUT(match, ip.ttl,
                                ipv4_key->ipv4_ttl, is_mask);
                SW_FLOW_KEY_PUT(match, ip.frag,
                                ipv4_key->ipv4_frag, is_mask);
                SW_FLOW_KEY_PUT(match, ipv4.addr.src,
                                ipv4_key->ipv4_src, is_mask);
                SW_FLOW_KEY_PUT(match, ipv4.addr.dst,
                                ipv4_key->ipv4_dst, is_mask);
                attrs &= ~(1 << OVS_KEY_ATTR_IPV4);
        }

        if (attrs & (1 << OVS_KEY_ATTR_IPV6)) {
                const struct ovs_key_ipv6 *ipv6_key;

                ipv6_key = nla_data(a[OVS_KEY_ATTR_IPV6]);
                if (!is_mask && ipv6_key->ipv6_frag > OVS_FRAG_TYPE_MAX) {
                        OVS_NLERR(log, "IPv6 frag type %d is out of range max %d",
                                  ipv6_key->ipv6_frag, OVS_FRAG_TYPE_MAX);
                        return -EINVAL;
                }

                if (!is_mask && ipv6_key->ipv6_label & htonl(0xFFF00000)) {
                        OVS_NLERR(log, "IPv6 flow label %x is out of range (max=%x)",
                                  ntohl(ipv6_key->ipv6_label), (1 << 20) - 1);
                        return -EINVAL;
                }

                SW_FLOW_KEY_PUT(match, ipv6.label,
                                ipv6_key->ipv6_label, is_mask);
                SW_FLOW_KEY_PUT(match, ip.proto,
                                ipv6_key->ipv6_proto, is_mask);
                SW_FLOW_KEY_PUT(match, ip.tos,
                                ipv6_key->ipv6_tclass, is_mask);
                SW_FLOW_KEY_PUT(match, ip.ttl,
                                ipv6_key->ipv6_hlimit, is_mask);
                SW_FLOW_KEY_PUT(match, ip.frag,
                                ipv6_key->ipv6_frag, is_mask);
                SW_FLOW_KEY_MEMCPY(match, ipv6.addr.src,
                                ipv6_key->ipv6_src,
                                sizeof(match->key->ipv6.addr.src),
                                is_mask);
                SW_FLOW_KEY_MEMCPY(match, ipv6.addr.dst,
                                ipv6_key->ipv6_dst,
                                sizeof(match->key->ipv6.addr.dst),
                                is_mask);

                attrs &= ~(1 << OVS_KEY_ATTR_IPV6);
        }

        if (attrs & (1ULL << OVS_KEY_ATTR_IPV6_EXTHDRS)) {
                const struct ovs_key_ipv6_exthdrs *ipv6_exthdrs_key;

                ipv6_exthdrs_key = nla_data(a[OVS_KEY_ATTR_IPV6_EXTHDRS]);

                SW_FLOW_KEY_PUT(match, ipv6.exthdrs,
                                ipv6_exthdrs_key->hdrs, is_mask);

                attrs &= ~(1ULL << OVS_KEY_ATTR_IPV6_EXTHDRS);
        }

        if (attrs & (1 << OVS_KEY_ATTR_ARP)) {
                const struct ovs_key_arp *arp_key;

                arp_key = nla_data(a[OVS_KEY_ATTR_ARP]);
                if (!is_mask && (arp_key->arp_op & htons(0xff00))) {
                        OVS_NLERR(log, "Unknown ARP opcode (opcode=%d).",
                                  arp_key->arp_op);
                        return -EINVAL;
                }

                SW_FLOW_KEY_PUT(match, ipv4.addr.src,
                                arp_key->arp_sip, is_mask);
                SW_FLOW_KEY_PUT(match, ipv4.addr.dst,
                        arp_key->arp_tip, is_mask);
                SW_FLOW_KEY_PUT(match, ip.proto,
                                ntohs(arp_key->arp_op), is_mask);
                SW_FLOW_KEY_MEMCPY(match, ipv4.arp.sha,
                                arp_key->arp_sha, ETH_ALEN, is_mask);
                SW_FLOW_KEY_MEMCPY(match, ipv4.arp.tha,
                                arp_key->arp_tha, ETH_ALEN, is_mask);

                attrs &= ~(1 << OVS_KEY_ATTR_ARP);
        }

        if (attrs & (1 << OVS_KEY_ATTR_NSH)) {
                if (nsh_key_put_from_nlattr(a[OVS_KEY_ATTR_NSH], match,
                                            is_mask, false, log) < 0)
                        return -EINVAL;
                attrs &= ~(1 << OVS_KEY_ATTR_NSH);
        }

        if (attrs & (1 << OVS_KEY_ATTR_MPLS)) {
                const struct ovs_key_mpls *mpls_key;
                u32 hdr_len;
                u32 label_count, label_count_mask, i;

                mpls_key = nla_data(a[OVS_KEY_ATTR_MPLS]);
                hdr_len = nla_len(a[OVS_KEY_ATTR_MPLS]);
                label_count = hdr_len / sizeof(struct ovs_key_mpls);

                if (label_count == 0 || label_count > MPLS_LABEL_DEPTH ||
                    hdr_len % sizeof(struct ovs_key_mpls))
                        return -EINVAL;

                label_count_mask =  GENMASK(label_count - 1, 0);

                for (i = 0 ; i < label_count; i++)
                        SW_FLOW_KEY_PUT(match, mpls.lse[i],
                                        mpls_key[i].mpls_lse, is_mask);

                SW_FLOW_KEY_PUT(match, mpls.num_labels_mask,
                                label_count_mask, is_mask);

                attrs &= ~(1 << OVS_KEY_ATTR_MPLS);
         }

        if (attrs & (1 << OVS_KEY_ATTR_TCP)) {
                const struct ovs_key_tcp *tcp_key;

                tcp_key = nla_data(a[OVS_KEY_ATTR_TCP]);
                SW_FLOW_KEY_PUT(match, tp.src, tcp_key->tcp_src, is_mask);
                SW_FLOW_KEY_PUT(match, tp.dst, tcp_key->tcp_dst, is_mask);
                attrs &= ~(1 << OVS_KEY_ATTR_TCP);
        }

        if (attrs & (1 << OVS_KEY_ATTR_TCP_FLAGS)) {
                SW_FLOW_KEY_PUT(match, tp.flags,
                                nla_get_be16(a[OVS_KEY_ATTR_TCP_FLAGS]),
                                is_mask);
                attrs &= ~(1 << OVS_KEY_ATTR_TCP_FLAGS);
        }

        if (attrs & (1 << OVS_KEY_ATTR_UDP)) {
                const struct ovs_key_udp *udp_key;

                udp_key = nla_data(a[OVS_KEY_ATTR_UDP]);
                SW_FLOW_KEY_PUT(match, tp.src, udp_key->udp_src, is_mask);
                SW_FLOW_KEY_PUT(match, tp.dst, udp_key->udp_dst, is_mask);
                attrs &= ~(1 << OVS_KEY_ATTR_UDP);
        }

        if (attrs & (1 << OVS_KEY_ATTR_SCTP)) {
                const struct ovs_key_sctp *sctp_key;

                sctp_key = nla_data(a[OVS_KEY_ATTR_SCTP]);
                SW_FLOW_KEY_PUT(match, tp.src, sctp_key->sctp_src, is_mask);
                SW_FLOW_KEY_PUT(match, tp.dst, sctp_key->sctp_dst, is_mask);
                attrs &= ~(1 << OVS_KEY_ATTR_SCTP);
        }

        if (attrs & (1 << OVS_KEY_ATTR_ICMP)) {
                const struct ovs_key_icmp *icmp_key;

                icmp_key = nla_data(a[OVS_KEY_ATTR_ICMP]);
                SW_FLOW_KEY_PUT(match, tp.src,
                                htons(icmp_key->icmp_type), is_mask);
                SW_FLOW_KEY_PUT(match, tp.dst,
                                htons(icmp_key->icmp_code), is_mask);
                attrs &= ~(1 << OVS_KEY_ATTR_ICMP);
        }

        if (attrs & (1 << OVS_KEY_ATTR_ICMPV6)) {
                const struct ovs_key_icmpv6 *icmpv6_key;

                icmpv6_key = nla_data(a[OVS_KEY_ATTR_ICMPV6]);
                SW_FLOW_KEY_PUT(match, tp.src,
                                htons(icmpv6_key->icmpv6_type), is_mask);
                SW_FLOW_KEY_PUT(match, tp.dst,
                                htons(icmpv6_key->icmpv6_code), is_mask);
                attrs &= ~(1 << OVS_KEY_ATTR_ICMPV6);
        }

        if (attrs & (1 << OVS_KEY_ATTR_ND)) {
                const struct ovs_key_nd *nd_key;

                nd_key = nla_data(a[OVS_KEY_ATTR_ND]);
                SW_FLOW_KEY_MEMCPY(match, ipv6.nd.target,
                        nd_key->nd_target,
                        sizeof(match->key->ipv6.nd.target),
                        is_mask);
                SW_FLOW_KEY_MEMCPY(match, ipv6.nd.sll,
                        nd_key->nd_sll, ETH_ALEN, is_mask);
                SW_FLOW_KEY_MEMCPY(match, ipv6.nd.tll,
                                nd_key->nd_tll, ETH_ALEN, is_mask);
                attrs &= ~(1 << OVS_KEY_ATTR_ND);
        }

        if (attrs != 0) {
                OVS_NLERR(log, "Unknown key attributes %llx",
                          (unsigned long long)attrs);
                return -EINVAL;
        }

        return 0;
}

static void nlattr_set(struct nlattr *attr, u8 val,
                       const struct ovs_len_tbl *tbl)
{
        struct nlattr *nla;
        int rem;

        /* The nlattr stream should already have been validated */
        nla_for_each_nested(nla, attr, rem) {
                if (tbl[nla_type(nla)].len == OVS_ATTR_NESTED)
                        nlattr_set(nla, val, tbl[nla_type(nla)].next ? : tbl);
                else
                        memset(nla_data(nla), val, nla_len(nla));

                if (nla_type(nla) == OVS_KEY_ATTR_CT_STATE)
                        *(u32 *)nla_data(nla) &= CT_SUPPORTED_MASK;
        }
}

static void mask_set_nlattr(struct nlattr *attr, u8 val)
{
        nlattr_set(attr, val, ovs_key_lens);
}

/**
 * ovs_nla_get_match - parses Netlink attributes into a flow key and
 * mask. In case the 'mask' is NULL, the flow is treated as exact match
 * flow. Otherwise, it is treated as a wildcarded flow, except the mask
 * does not include any don't care bit.
 * @net: Used to determine per-namespace field support.
 * @match: receives the extracted flow match information.
 * @nla_key: Netlink attribute holding nested %OVS_KEY_ATTR_* Netlink attribute
 * sequence. The fields should of the packet that triggered the creation
 * of this flow.
 * @nla_mask: Optional. Netlink attribute holding nested %OVS_KEY_ATTR_*
 * Netlink attribute specifies the mask field of the wildcarded flow.
 * @log: Boolean to allow kernel error logging.  Normally true, but when
 * probing for feature compatibility this should be passed in as false to
 * suppress unnecessary error logging.
 */
int ovs_nla_get_match(struct net *net, struct sw_flow_match *match,
                      const struct nlattr *nla_key,
                      const struct nlattr *nla_mask,
                      bool log)
{
        const struct nlattr *a[OVS_KEY_ATTR_MAX + 1];
        struct nlattr *newmask = NULL;
        u64 key_attrs = 0;
        u64 mask_attrs = 0;
        int err;

        err = parse_flow_nlattrs(nla_key, a, &key_attrs, log);
        if (err)
                return err;

        err = parse_vlan_from_nlattrs(match, &key_attrs, a, false, log);
        if (err)
                return err;

        err = ovs_key_from_nlattrs(net, match, key_attrs, a, false, log);
        if (err)
                return err;

        if (match->mask) {
                if (!nla_mask) {
                        /* Create an exact match mask. We need to set to 0xff
                         * all the 'match->mask' fields that have been touched
                         * in 'match->key'. We cannot simply memset
                         * 'match->mask', because padding bytes and fields not
                         * specified in 'match->key' should be left to 0.
                         * Instead, we use a stream of netlink attributes,
                         * copied from 'key' and set to 0xff.
                         * ovs_key_from_nlattrs() will take care of filling
                         * 'match->mask' appropriately.
                         */
                        newmask = kmemdup(nla_key,
                                          nla_total_size(nla_len(nla_key)),
                                          GFP_KERNEL);
                        if (!newmask)
                                return -ENOMEM;

                        mask_set_nlattr(newmask, 0xff);

                        /* The userspace does not send tunnel attributes that
                         * are 0, but we should not wildcard them nonetheless.
                         */
                        if (match->key->tun_proto)
                                SW_FLOW_KEY_MEMSET_FIELD(match, tun_key,
                                                         0xff, true);

                        nla_mask = newmask;
                }

                err = parse_flow_mask_nlattrs(nla_mask, a, &mask_attrs, log);
                if (err)
                        goto free_newmask;

                /* Always match on tci. */
                SW_FLOW_KEY_PUT(match, eth.vlan.tci, htons(0xffff), true);
                SW_FLOW_KEY_PUT(match, eth.cvlan.tci, htons(0xffff), true);

                err = parse_vlan_from_nlattrs(match, &mask_attrs, a, true, log);
                if (err)
                        goto free_newmask;

                err = ovs_key_from_nlattrs(net, match, mask_attrs, a, true,
                                           log);
                if (err)
                        goto free_newmask;
        }

        if (!match_validate(match, key_attrs, mask_attrs, log))
                err = -EINVAL;

free_newmask:
        kfree(newmask);
        return err;
}

static size_t get_ufid_len(const struct nlattr *attr, bool log)
{
        size_t len;

        if (!attr)
                return 0;

        len = nla_len(attr);
        if (len < 1 || len > MAX_UFID_LENGTH) {
                OVS_NLERR(log, "ufid size %u bytes exceeds the range (1, %d)",
                          nla_len(attr), MAX_UFID_LENGTH);
                return 0;
        }

        return len;
}

/* Initializes 'flow->ufid', returning true if 'attr' contains a valid UFID,
 * or false otherwise.
 */
bool ovs_nla_get_ufid(struct sw_flow_id *sfid, const struct nlattr *attr,
                      bool log)
{
        sfid->ufid_len = get_ufid_len(attr, log);
        if (sfid->ufid_len)
                memcpy(sfid->ufid, nla_data(attr), sfid->ufid_len);

        return sfid->ufid_len;
}

int ovs_nla_get_identifier(struct sw_flow_id *sfid, const struct nlattr *ufid,
                           const struct sw_flow_key *key, bool log)
{
        struct sw_flow_key *new_key;

        if (ovs_nla_get_ufid(sfid, ufid, log))
                return 0;

        /* If UFID was not provided, use unmasked key. */
        new_key = kmalloc_obj(*new_key);
        if (!new_key)
                return -ENOMEM;
        memcpy(new_key, key, sizeof(*key));
        sfid->unmasked_key = new_key;

        return 0;
}

u32 ovs_nla_get_ufid_flags(const struct nlattr *attr)
{
        return nla_get_u32_default(attr, 0);
}

/**
 * ovs_nla_get_flow_metadata - parses Netlink attributes into a flow key.
 * @net: Network namespace.
 * @key: Receives extracted in_port, priority, tun_key, skb_mark and conntrack
 * metadata.
 * @a: Array of netlink attributes holding parsed %OVS_KEY_ATTR_* Netlink
 * attributes.
 * @attrs: Bit mask for the netlink attributes included in @a.
 * @log: Boolean to allow kernel error logging.  Normally true, but when
 * probing for feature compatibility this should be passed in as false to
 * suppress unnecessary error logging.
 *
 * This parses a series of Netlink attributes that form a flow key, which must
 * take the same form accepted by flow_from_nlattrs(), but only enough of it to
 * get the metadata, that is, the parts of the flow key that cannot be
 * extracted from the packet itself.
 *
 * This must be called before the packet key fields are filled in 'key'.
 */

int ovs_nla_get_flow_metadata(struct net *net,
                              const struct nlattr *a[OVS_KEY_ATTR_MAX + 1],
                              u64 attrs, struct sw_flow_key *key, bool log)
{
        struct sw_flow_match match;

        memset(&match, 0, sizeof(match));
        match.key = key;

        key->ct_state = 0;
        key->ct_zone = 0;
        key->ct_orig_proto = 0;
        memset(&key->ct, 0, sizeof(key->ct));
        memset(&key->ipv4.ct_orig, 0, sizeof(key->ipv4.ct_orig));
        memset(&key->ipv6.ct_orig, 0, sizeof(key->ipv6.ct_orig));

        key->phy.in_port = DP_MAX_PORTS;

        return metadata_from_nlattrs(net, &match, &attrs, a, false, log);
}

static int ovs_nla_put_vlan(struct sk_buff *skb, const struct vlan_head *vh,
                            bool is_mask)
{
        __be16 eth_type = !is_mask ? vh->tpid : htons(0xffff);

        if (nla_put_be16(skb, OVS_KEY_ATTR_ETHERTYPE, eth_type) ||
            nla_put_be16(skb, OVS_KEY_ATTR_VLAN, vh->tci))
                return -EMSGSIZE;
        return 0;
}

static int nsh_key_to_nlattr(const struct ovs_key_nsh *nsh, bool is_mask,
                             struct sk_buff *skb)
{
        struct nlattr *start;

        start = nla_nest_start_noflag(skb, OVS_KEY_ATTR_NSH);
        if (!start)
                return -EMSGSIZE;

        if (nla_put(skb, OVS_NSH_KEY_ATTR_BASE, sizeof(nsh->base), &nsh->base))
                goto nla_put_failure;

        if (is_mask || nsh->base.mdtype == NSH_M_TYPE1) {
                if (nla_put(skb, OVS_NSH_KEY_ATTR_MD1,
                            sizeof(nsh->context), nsh->context))
                        goto nla_put_failure;
        }

        /* Don't support MD type 2 yet */

        nla_nest_end(skb, start);

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int __ovs_nla_put_key(const struct sw_flow_key *swkey,
                             const struct sw_flow_key *output, bool is_mask,
                             struct sk_buff *skb)
{
        struct ovs_key_ethernet *eth_key;
        struct nlattr *nla;
        struct nlattr *encap = NULL;
        struct nlattr *in_encap = NULL;

        if (nla_put_u32(skb, OVS_KEY_ATTR_RECIRC_ID, output->recirc_id))
                goto nla_put_failure;

        if (nla_put_u32(skb, OVS_KEY_ATTR_DP_HASH, output->ovs_flow_hash))
                goto nla_put_failure;

        if (nla_put_u32(skb, OVS_KEY_ATTR_PRIORITY, output->phy.priority))
                goto nla_put_failure;

        if ((swkey->tun_proto || is_mask)) {
                const void *opts = NULL;

                if (ip_tunnel_is_options_present(output->tun_key.tun_flags))
                        opts = TUN_METADATA_OPTS(output, swkey->tun_opts_len);

                if (ip_tun_to_nlattr(skb, &output->tun_key, opts,
                                     swkey->tun_opts_len, swkey->tun_proto, 0))
                        goto nla_put_failure;
        }

        if (swkey->phy.in_port == DP_MAX_PORTS) {
                if (is_mask && (output->phy.in_port == 0xffff))
                        if (nla_put_u32(skb, OVS_KEY_ATTR_IN_PORT, 0xffffffff))
                                goto nla_put_failure;
        } else {
                u16 upper_u16;
                upper_u16 = !is_mask ? 0 : 0xffff;

                if (nla_put_u32(skb, OVS_KEY_ATTR_IN_PORT,
                                (upper_u16 << 16) | output->phy.in_port))
                        goto nla_put_failure;
        }

        if (nla_put_u32(skb, OVS_KEY_ATTR_SKB_MARK, output->phy.skb_mark))
                goto nla_put_failure;

        if (ovs_ct_put_key(swkey, output, skb))
                goto nla_put_failure;

        if (ovs_key_mac_proto(swkey) == MAC_PROTO_ETHERNET) {
                nla = nla_reserve(skb, OVS_KEY_ATTR_ETHERNET, sizeof(*eth_key));
                if (!nla)
                        goto nla_put_failure;

                eth_key = nla_data(nla);
                ether_addr_copy(eth_key->eth_src, output->eth.src);
                ether_addr_copy(eth_key->eth_dst, output->eth.dst);

                if (swkey->eth.vlan.tci || eth_type_vlan(swkey->eth.type)) {
                        if (ovs_nla_put_vlan(skb, &output->eth.vlan, is_mask))
                                goto nla_put_failure;
                        encap = nla_nest_start_noflag(skb, OVS_KEY_ATTR_ENCAP);
                        if (!swkey->eth.vlan.tci)
                                goto unencap;

                        if (swkey->eth.cvlan.tci || eth_type_vlan(swkey->eth.type)) {
                                if (ovs_nla_put_vlan(skb, &output->eth.cvlan, is_mask))
                                        goto nla_put_failure;
                                in_encap = nla_nest_start_noflag(skb,
                                                                 OVS_KEY_ATTR_ENCAP);
                                if (!swkey->eth.cvlan.tci)
                                        goto unencap;
                        }
                }

                if (swkey->eth.type == htons(ETH_P_802_2)) {
                        /*
                        * Ethertype 802.2 is represented in the netlink with omitted
                        * OVS_KEY_ATTR_ETHERTYPE in the flow key attribute, and
                        * 0xffff in the mask attribute.  Ethertype can also
                        * be wildcarded.
                        */
                        if (is_mask && output->eth.type)
                                if (nla_put_be16(skb, OVS_KEY_ATTR_ETHERTYPE,
                                                        output->eth.type))
                                        goto nla_put_failure;
                        goto unencap;
                }
        }

        if (nla_put_be16(skb, OVS_KEY_ATTR_ETHERTYPE, output->eth.type))
                goto nla_put_failure;

        if (eth_type_vlan(swkey->eth.type)) {
                /* There are 3 VLAN tags, we don't know anything about the rest
                 * of the packet, so truncate here.
                 */
                WARN_ON_ONCE(!(encap && in_encap));
                goto unencap;
        }

        if (swkey->eth.type == htons(ETH_P_IP)) {
                struct ovs_key_ipv4 *ipv4_key;

                nla = nla_reserve(skb, OVS_KEY_ATTR_IPV4, sizeof(*ipv4_key));
                if (!nla)
                        goto nla_put_failure;
                ipv4_key = nla_data(nla);
                ipv4_key->ipv4_src = output->ipv4.addr.src;
                ipv4_key->ipv4_dst = output->ipv4.addr.dst;
                ipv4_key->ipv4_proto = output->ip.proto;
                ipv4_key->ipv4_tos = output->ip.tos;
                ipv4_key->ipv4_ttl = output->ip.ttl;
                ipv4_key->ipv4_frag = output->ip.frag;
        } else if (swkey->eth.type == htons(ETH_P_IPV6)) {
                struct ovs_key_ipv6 *ipv6_key;
                struct ovs_key_ipv6_exthdrs *ipv6_exthdrs_key;

                nla = nla_reserve(skb, OVS_KEY_ATTR_IPV6, sizeof(*ipv6_key));
                if (!nla)
                        goto nla_put_failure;
                ipv6_key = nla_data(nla);
                memcpy(ipv6_key->ipv6_src, &output->ipv6.addr.src,
                                sizeof(ipv6_key->ipv6_src));
                memcpy(ipv6_key->ipv6_dst, &output->ipv6.addr.dst,
                                sizeof(ipv6_key->ipv6_dst));
                ipv6_key->ipv6_label = output->ipv6.label;
                ipv6_key->ipv6_proto = output->ip.proto;
                ipv6_key->ipv6_tclass = output->ip.tos;
                ipv6_key->ipv6_hlimit = output->ip.ttl;
                ipv6_key->ipv6_frag = output->ip.frag;

                nla = nla_reserve(skb, OVS_KEY_ATTR_IPV6_EXTHDRS,
                                  sizeof(*ipv6_exthdrs_key));
                if (!nla)
                        goto nla_put_failure;
                ipv6_exthdrs_key = nla_data(nla);
                ipv6_exthdrs_key->hdrs = output->ipv6.exthdrs;
        } else if (swkey->eth.type == htons(ETH_P_NSH)) {
                if (nsh_key_to_nlattr(&output->nsh, is_mask, skb))
                        goto nla_put_failure;
        } else if (swkey->eth.type == htons(ETH_P_ARP) ||
                   swkey->eth.type == htons(ETH_P_RARP)) {
                struct ovs_key_arp *arp_key;

                nla = nla_reserve(skb, OVS_KEY_ATTR_ARP, sizeof(*arp_key));
                if (!nla)
                        goto nla_put_failure;
                arp_key = nla_data(nla);
                memset(arp_key, 0, sizeof(struct ovs_key_arp));
                arp_key->arp_sip = output->ipv4.addr.src;
                arp_key->arp_tip = output->ipv4.addr.dst;
                arp_key->arp_op = htons(output->ip.proto);
                ether_addr_copy(arp_key->arp_sha, output->ipv4.arp.sha);
                ether_addr_copy(arp_key->arp_tha, output->ipv4.arp.tha);
        } else if (eth_p_mpls(swkey->eth.type)) {
                u8 i, num_labels;
                struct ovs_key_mpls *mpls_key;

                num_labels = hweight_long(output->mpls.num_labels_mask);
                nla = nla_reserve(skb, OVS_KEY_ATTR_MPLS,
                                  num_labels * sizeof(*mpls_key));
                if (!nla)
                        goto nla_put_failure;

                mpls_key = nla_data(nla);
                for (i = 0; i < num_labels; i++)
                        mpls_key[i].mpls_lse = output->mpls.lse[i];
        }

        if ((swkey->eth.type == htons(ETH_P_IP) ||
             swkey->eth.type == htons(ETH_P_IPV6)) &&
             swkey->ip.frag != OVS_FRAG_TYPE_LATER) {

                if (swkey->ip.proto == IPPROTO_TCP) {
                        struct ovs_key_tcp *tcp_key;

                        nla = nla_reserve(skb, OVS_KEY_ATTR_TCP, sizeof(*tcp_key));
                        if (!nla)
                                goto nla_put_failure;
                        tcp_key = nla_data(nla);
                        tcp_key->tcp_src = output->tp.src;
                        tcp_key->tcp_dst = output->tp.dst;
                        if (nla_put_be16(skb, OVS_KEY_ATTR_TCP_FLAGS,
                                         output->tp.flags))
                                goto nla_put_failure;
                } else if (swkey->ip.proto == IPPROTO_UDP) {
                        struct ovs_key_udp *udp_key;

                        nla = nla_reserve(skb, OVS_KEY_ATTR_UDP, sizeof(*udp_key));
                        if (!nla)
                                goto nla_put_failure;
                        udp_key = nla_data(nla);
                        udp_key->udp_src = output->tp.src;
                        udp_key->udp_dst = output->tp.dst;
                } else if (swkey->ip.proto == IPPROTO_SCTP) {
                        struct ovs_key_sctp *sctp_key;

                        nla = nla_reserve(skb, OVS_KEY_ATTR_SCTP, sizeof(*sctp_key));
                        if (!nla)
                                goto nla_put_failure;
                        sctp_key = nla_data(nla);
                        sctp_key->sctp_src = output->tp.src;
                        sctp_key->sctp_dst = output->tp.dst;
                } else if (swkey->eth.type == htons(ETH_P_IP) &&
                           swkey->ip.proto == IPPROTO_ICMP) {
                        struct ovs_key_icmp *icmp_key;

                        nla = nla_reserve(skb, OVS_KEY_ATTR_ICMP, sizeof(*icmp_key));
                        if (!nla)
                                goto nla_put_failure;
                        icmp_key = nla_data(nla);
                        icmp_key->icmp_type = ntohs(output->tp.src);
                        icmp_key->icmp_code = ntohs(output->tp.dst);
                } else if (swkey->eth.type == htons(ETH_P_IPV6) &&
                           swkey->ip.proto == IPPROTO_ICMPV6) {
                        struct ovs_key_icmpv6 *icmpv6_key;

                        nla = nla_reserve(skb, OVS_KEY_ATTR_ICMPV6,
                                                sizeof(*icmpv6_key));
                        if (!nla)
                                goto nla_put_failure;
                        icmpv6_key = nla_data(nla);
                        icmpv6_key->icmpv6_type = ntohs(output->tp.src);
                        icmpv6_key->icmpv6_code = ntohs(output->tp.dst);

                        if (swkey->tp.src == htons(NDISC_NEIGHBOUR_SOLICITATION) ||
                            swkey->tp.src == htons(NDISC_NEIGHBOUR_ADVERTISEMENT)) {
                                struct ovs_key_nd *nd_key;

                                nla = nla_reserve(skb, OVS_KEY_ATTR_ND, sizeof(*nd_key));
                                if (!nla)
                                        goto nla_put_failure;
                                nd_key = nla_data(nla);
                                memcpy(nd_key->nd_target, &output->ipv6.nd.target,
                                                        sizeof(nd_key->nd_target));
                                ether_addr_copy(nd_key->nd_sll, output->ipv6.nd.sll);
                                ether_addr_copy(nd_key->nd_tll, output->ipv6.nd.tll);
                        }
                }
        }

unencap:
        if (in_encap)
                nla_nest_end(skb, in_encap);
        if (encap)
                nla_nest_end(skb, encap);

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

int ovs_nla_put_key(const struct sw_flow_key *swkey,
                    const struct sw_flow_key *output, int attr, bool is_mask,
                    struct sk_buff *skb)
{
        int err;
        struct nlattr *nla;

        nla = nla_nest_start_noflag(skb, attr);
        if (!nla)
                return -EMSGSIZE;
        err = __ovs_nla_put_key(swkey, output, is_mask, skb);
        if (err)
                return err;
        nla_nest_end(skb, nla);

        return 0;
}

/* Called with ovs_mutex or RCU read lock. */
int ovs_nla_put_identifier(const struct sw_flow *flow, struct sk_buff *skb)
{
        if (ovs_identifier_is_ufid(&flow->id))
                return nla_put(skb, OVS_FLOW_ATTR_UFID, flow->id.ufid_len,
                               flow->id.ufid);

        return ovs_nla_put_key(flow->id.unmasked_key, flow->id.unmasked_key,
                               OVS_FLOW_ATTR_KEY, false, skb);
}

/* Called with ovs_mutex or RCU read lock. */
int ovs_nla_put_masked_key(const struct sw_flow *flow, struct sk_buff *skb)
{
        return ovs_nla_put_key(&flow->key, &flow->key,
                                OVS_FLOW_ATTR_KEY, false, skb);
}

/* Called with ovs_mutex or RCU read lock. */
int ovs_nla_put_mask(const struct sw_flow *flow, struct sk_buff *skb)
{
        return ovs_nla_put_key(&flow->key, &flow->mask->key,
                                OVS_FLOW_ATTR_MASK, true, skb);
}

static struct sw_flow_actions *nla_alloc_flow_actions(int size)
{
        struct sw_flow_actions *sfa;

        sfa = kmalloc(kmalloc_size_roundup(sizeof(*sfa) + size), GFP_KERNEL);
        if (!sfa)
                return ERR_PTR(-ENOMEM);

        sfa->actions_len = 0;
        return sfa;
}

static void ovs_nla_free_nested_actions(const struct nlattr *actions, int len);

static void ovs_nla_free_check_pkt_len_action(const struct nlattr *action)
{
        const struct nlattr *a;
        int rem;

        nla_for_each_nested(a, action, rem) {
                switch (nla_type(a)) {
                case OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL:
                case OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER:
                        ovs_nla_free_nested_actions(nla_data(a), nla_len(a));
                        break;
                }
        }
}

static void ovs_nla_free_clone_action(const struct nlattr *action)
{
        const struct nlattr *a = nla_data(action);
        int rem = nla_len(action);

        switch (nla_type(a)) {
        case OVS_CLONE_ATTR_EXEC:
                /* The real list of actions follows this attribute. */
                a = nla_next(a, &rem);
                ovs_nla_free_nested_actions(a, rem);
                break;
        }
}

static void ovs_nla_free_dec_ttl_action(const struct nlattr *action)
{
        const struct nlattr *a = nla_data(action);

        switch (nla_type(a)) {
        case OVS_DEC_TTL_ATTR_ACTION:
                ovs_nla_free_nested_actions(nla_data(a), nla_len(a));
                break;
        }
}

static void ovs_nla_free_sample_action(const struct nlattr *action)
{
        const struct nlattr *a = nla_data(action);
        int rem = nla_len(action);

        switch (nla_type(a)) {
        case OVS_SAMPLE_ATTR_ARG:
                /* The real list of actions follows this attribute. */
                a = nla_next(a, &rem);
                ovs_nla_free_nested_actions(a, rem);
                break;
        }
}

static void ovs_nla_free_set_action(const struct nlattr *a)
{
        const struct nlattr *ovs_key = nla_data(a);
        struct ovs_tunnel_info *ovs_tun;

        switch (nla_type(ovs_key)) {
        case OVS_KEY_ATTR_TUNNEL_INFO:
                ovs_tun = nla_data(ovs_key);
                dst_release((struct dst_entry *)ovs_tun->tun_dst);
                break;
        }
}

static void ovs_nla_free_nested_actions(const struct nlattr *actions, int len)
{
        const struct nlattr *a;
        int rem;

        /* Whenever new actions are added, the need to update this
         * function should be considered.
         */
        BUILD_BUG_ON(OVS_ACTION_ATTR_MAX != 25);

        if (!actions)
                return;

        nla_for_each_attr(a, actions, len, rem) {
                switch (nla_type(a)) {
                case OVS_ACTION_ATTR_CHECK_PKT_LEN:
                        ovs_nla_free_check_pkt_len_action(a);
                        break;

                case OVS_ACTION_ATTR_CLONE:
                        ovs_nla_free_clone_action(a);
                        break;

                case OVS_ACTION_ATTR_CT:
                        ovs_ct_free_action(a);
                        break;

                case OVS_ACTION_ATTR_DEC_TTL:
                        ovs_nla_free_dec_ttl_action(a);
                        break;

                case OVS_ACTION_ATTR_SAMPLE:
                        ovs_nla_free_sample_action(a);
                        break;

                case OVS_ACTION_ATTR_SET:
                        ovs_nla_free_set_action(a);
                        break;
                }
        }
}

void ovs_nla_free_flow_actions(struct sw_flow_actions *sf_acts)
{
        if (!sf_acts)
                return;

        ovs_nla_free_nested_actions(sf_acts->actions, sf_acts->actions_len);
        kfree(sf_acts);
}

static void __ovs_nla_free_flow_actions(struct rcu_head *head)
{
        ovs_nla_free_flow_actions(container_of(head, struct sw_flow_actions, rcu));
}

/* Schedules 'sf_acts' to be freed after the next RCU grace period.
 * The caller must hold rcu_read_lock for this to be sensible. */
void ovs_nla_free_flow_actions_rcu(struct sw_flow_actions *sf_acts)
{
        call_rcu(&sf_acts->rcu, __ovs_nla_free_flow_actions);
}

static struct nlattr *reserve_sfa_size(struct sw_flow_actions **sfa,
                                       int attr_len, bool log)
{

        struct sw_flow_actions *acts;
        int new_acts_size;
        size_t req_size = NLA_ALIGN(attr_len);
        int next_offset = offsetof(struct sw_flow_actions, actions) +
                                        (*sfa)->actions_len;

        if (req_size <= (ksize(*sfa) - next_offset))
                goto out;

        new_acts_size = max(next_offset + req_size, ksize(*sfa) * 2);

        acts = nla_alloc_flow_actions(new_acts_size);
        if (IS_ERR(acts))
                return ERR_CAST(acts);

        memcpy(acts->actions, (*sfa)->actions, (*sfa)->actions_len);
        acts->actions_len = (*sfa)->actions_len;
        acts->orig_len = (*sfa)->orig_len;
        kfree(*sfa);
        *sfa = acts;

out:
        (*sfa)->actions_len += req_size;
        return  (struct nlattr *) ((unsigned char *)(*sfa) + next_offset);
}

static struct nlattr *__add_action(struct sw_flow_actions **sfa,
                                   int attrtype, void *data, int len, bool log)
{
        struct nlattr *a;

        a = reserve_sfa_size(sfa, nla_attr_size(len), log);
        if (IS_ERR(a))
                return a;

        a->nla_type = attrtype;
        a->nla_len = nla_attr_size(len);

        if (data)
                memcpy(nla_data(a), data, len);
        memset((unsigned char *) a + a->nla_len, 0, nla_padlen(len));

        return a;
}

int ovs_nla_add_action(struct sw_flow_actions **sfa, int attrtype, void *data,
                       int len, bool log)
{
        struct nlattr *a;

        a = __add_action(sfa, attrtype, data, len, log);

        return PTR_ERR_OR_ZERO(a);
}

static inline int add_nested_action_start(struct sw_flow_actions **sfa,
                                          int attrtype, bool log)
{
        int used = (*sfa)->actions_len;
        int err;

        err = ovs_nla_add_action(sfa, attrtype, NULL, 0, log);
        if (err)
                return err;

        return used;
}

static inline void add_nested_action_end(struct sw_flow_actions *sfa,
                                         int st_offset)
{
        struct nlattr *a = (struct nlattr *) ((unsigned char *)sfa->actions +
                                                               st_offset);

        a->nla_len = sfa->actions_len - st_offset;
}

static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
                                  const struct sw_flow_key *key,
                                  struct sw_flow_actions **sfa,
                                  __be16 eth_type, __be16 vlan_tci,
                                  u32 mpls_label_count, bool log,
                                  u32 depth);

static int validate_and_copy_sample(struct net *net, const struct nlattr *attr,
                                    const struct sw_flow_key *key,
                                    struct sw_flow_actions **sfa,
                                    __be16 eth_type, __be16 vlan_tci,
                                    u32 mpls_label_count, bool log, bool last,
                                    u32 depth)
{
        const struct nlattr *attrs[OVS_SAMPLE_ATTR_MAX + 1];
        const struct nlattr *probability, *actions;
        const struct nlattr *a;
        int rem, start, err;
        struct sample_arg arg;

        memset(attrs, 0, sizeof(attrs));
        nla_for_each_nested(a, attr, rem) {
                int type = nla_type(a);
                if (!type || type > OVS_SAMPLE_ATTR_MAX || attrs[type])
                        return -EINVAL;
                attrs[type] = a;
        }
        if (rem)
                return -EINVAL;

        probability = attrs[OVS_SAMPLE_ATTR_PROBABILITY];
        if (!probability || nla_len(probability) != sizeof(u32))
                return -EINVAL;

        actions = attrs[OVS_SAMPLE_ATTR_ACTIONS];
        if (!actions || (nla_len(actions) && nla_len(actions) < NLA_HDRLEN))
                return -EINVAL;

        /* validation done, copy sample action. */
        start = add_nested_action_start(sfa, OVS_ACTION_ATTR_SAMPLE, log);
        if (start < 0)
                return start;

        /* When both skb and flow may be changed, put the sample
         * into a deferred fifo. On the other hand, if only skb
         * may be modified, the actions can be executed in place.
         *
         * Do this analysis at the flow installation time.
         * Set 'clone_action->exec' to true if the actions can be
         * executed without being deferred.
         *
         * If the sample is the last action, it can always be excuted
         * rather than deferred.
         */
        arg.exec = last || !actions_may_change_flow(actions);
        arg.probability = nla_get_u32(probability);

        err = ovs_nla_add_action(sfa, OVS_SAMPLE_ATTR_ARG, &arg, sizeof(arg),
                                 log);
        if (err)
                return err;

        err = __ovs_nla_copy_actions(net, actions, key, sfa,
                                     eth_type, vlan_tci, mpls_label_count, log,
                                     depth + 1);

        if (err)
                return err;

        add_nested_action_end(*sfa, start);

        return 0;
}

static int validate_and_copy_dec_ttl(struct net *net,
                                     const struct nlattr *attr,
                                     const struct sw_flow_key *key,
                                     struct sw_flow_actions **sfa,
                                     __be16 eth_type, __be16 vlan_tci,
                                     u32 mpls_label_count, bool log,
                                     u32 depth)
{
        const struct nlattr *attrs[OVS_DEC_TTL_ATTR_MAX + 1];
        int start, action_start, err, rem;
        const struct nlattr *a, *actions;

        memset(attrs, 0, sizeof(attrs));
        nla_for_each_nested(a, attr, rem) {
                int type = nla_type(a);

                /* Ignore unknown attributes to be future proof. */
                if (type > OVS_DEC_TTL_ATTR_MAX)
                        continue;

                if (!type || attrs[type]) {
                        OVS_NLERR(log, "Duplicate or invalid key (type %d).",
                                  type);
                        return -EINVAL;
                }

                attrs[type] = a;
        }

        if (rem) {
                OVS_NLERR(log, "Message has %d unknown bytes.", rem);
                return -EINVAL;
        }

        actions = attrs[OVS_DEC_TTL_ATTR_ACTION];
        if (!actions || (nla_len(actions) && nla_len(actions) < NLA_HDRLEN)) {
                OVS_NLERR(log, "Missing valid actions attribute.");
                return -EINVAL;
        }

        start = add_nested_action_start(sfa, OVS_ACTION_ATTR_DEC_TTL, log);
        if (start < 0)
                return start;

        action_start = add_nested_action_start(sfa, OVS_DEC_TTL_ATTR_ACTION, log);
        if (action_start < 0)
                return action_start;

        err = __ovs_nla_copy_actions(net, actions, key, sfa, eth_type,
                                     vlan_tci, mpls_label_count, log,
                                     depth + 1);
        if (err)
                return err;

        add_nested_action_end(*sfa, action_start);
        add_nested_action_end(*sfa, start);
        return 0;
}

static int validate_and_copy_clone(struct net *net,
                                   const struct nlattr *attr,
                                   const struct sw_flow_key *key,
                                   struct sw_flow_actions **sfa,
                                   __be16 eth_type, __be16 vlan_tci,
                                   u32 mpls_label_count, bool log, bool last,
                                   u32 depth)
{
        int start, err;
        u32 exec;

        if (nla_len(attr) && nla_len(attr) < NLA_HDRLEN)
                return -EINVAL;

        start = add_nested_action_start(sfa, OVS_ACTION_ATTR_CLONE, log);
        if (start < 0)
                return start;

        exec = last || !actions_may_change_flow(attr);

        err = ovs_nla_add_action(sfa, OVS_CLONE_ATTR_EXEC, &exec,
                                 sizeof(exec), log);
        if (err)
                return err;

        err = __ovs_nla_copy_actions(net, attr, key, sfa,
                                     eth_type, vlan_tci, mpls_label_count, log,
                                     depth + 1);
        if (err)
                return err;

        add_nested_action_end(*sfa, start);

        return 0;
}

void ovs_match_init(struct sw_flow_match *match,
                    struct sw_flow_key *key,
                    bool reset_key,
                    struct sw_flow_mask *mask)
{
        memset(match, 0, sizeof(*match));
        match->key = key;
        match->mask = mask;

        if (reset_key)
                memset(key, 0, sizeof(*key));

        if (mask) {
                memset(&mask->key, 0, sizeof(mask->key));
                mask->range.start = mask->range.end = 0;
        }
}

static int validate_geneve_opts(struct sw_flow_key *key)
{
        struct geneve_opt *option;
        int opts_len = key->tun_opts_len;
        bool crit_opt = false;

        option = (struct geneve_opt *)TUN_METADATA_OPTS(key, key->tun_opts_len);
        while (opts_len > 0) {
                int len;

                if (opts_len < sizeof(*option))
                        return -EINVAL;

                len = sizeof(*option) + option->length * 4;
                if (len > opts_len)
                        return -EINVAL;

                crit_opt |= !!(option->type & GENEVE_CRIT_OPT_TYPE);

                option = (struct geneve_opt *)((u8 *)option + len);
                opts_len -= len;
        }

        if (crit_opt)
                __set_bit(IP_TUNNEL_CRIT_OPT_BIT, key->tun_key.tun_flags);

        return 0;
}

static int validate_and_copy_set_tun(const struct nlattr *attr,
                                     struct sw_flow_actions **sfa, bool log)
{
        IP_TUNNEL_DECLARE_FLAGS(dst_opt_type) = { };
        struct sw_flow_match match;
        struct sw_flow_key key;
        struct metadata_dst *tun_dst;
        struct ip_tunnel_info *tun_info;
        struct ovs_tunnel_info *ovs_tun;
        struct nlattr *a;
        int err = 0, start, opts_type;

        ovs_match_init(&match, &key, true, NULL);
        opts_type = ip_tun_from_nlattr(nla_data(attr), &match, false, log);
        if (opts_type < 0)
                return opts_type;

        if (key.tun_opts_len) {
                switch (opts_type) {
                case OVS_TUNNEL_KEY_ATTR_GENEVE_OPTS:
                        err = validate_geneve_opts(&key);
                        if (err < 0)
                                return err;

                        __set_bit(IP_TUNNEL_GENEVE_OPT_BIT, dst_opt_type);
                        break;
                case OVS_TUNNEL_KEY_ATTR_VXLAN_OPTS:
                        __set_bit(IP_TUNNEL_VXLAN_OPT_BIT, dst_opt_type);
                        break;
                case OVS_TUNNEL_KEY_ATTR_ERSPAN_OPTS:
                        __set_bit(IP_TUNNEL_ERSPAN_OPT_BIT, dst_opt_type);
                        break;
                }
        }

        start = add_nested_action_start(sfa, OVS_ACTION_ATTR_SET, log);
        if (start < 0)
                return start;

        tun_dst = metadata_dst_alloc(key.tun_opts_len, METADATA_IP_TUNNEL,
                                     GFP_KERNEL);

        if (!tun_dst)
                return -ENOMEM;

        err = dst_cache_init(&tun_dst->u.tun_info.dst_cache, GFP_KERNEL);
        if (err) {
                dst_release((struct dst_entry *)tun_dst);
                return err;
        }

        a = __add_action(sfa, OVS_KEY_ATTR_TUNNEL_INFO, NULL,
                         sizeof(*ovs_tun), log);
        if (IS_ERR(a)) {
                dst_release((struct dst_entry *)tun_dst);
                return PTR_ERR(a);
        }

        ovs_tun = nla_data(a);
        ovs_tun->tun_dst = tun_dst;

        tun_info = &tun_dst->u.tun_info;
        tun_info->mode = IP_TUNNEL_INFO_TX;
        if (key.tun_proto == AF_INET6)
                tun_info->mode |= IP_TUNNEL_INFO_IPV6;
        else if (key.tun_proto == AF_INET && key.tun_key.u.ipv4.dst == 0)
                tun_info->mode |= IP_TUNNEL_INFO_BRIDGE;
        tun_info->key = key.tun_key;

        /* We need to store the options in the action itself since
         * everything else will go away after flow setup. We can append
         * it to tun_info and then point there.
         */
        ip_tunnel_info_opts_set(tun_info,
                                TUN_METADATA_OPTS(&key, key.tun_opts_len),
                                key.tun_opts_len, dst_opt_type);
        add_nested_action_end(*sfa, start);

        return err;
}

static bool validate_push_nsh(const struct nlattr *a, bool log)
{
        struct nlattr *nsh_key = nla_data(a);
        struct sw_flow_match match;
        struct sw_flow_key key;

        /* There must be one and only one NSH header. */
        if (!nla_ok(nsh_key, nla_len(a)) ||
            nla_total_size(nla_len(nsh_key)) != nla_len(a) ||
            nla_type(nsh_key) != OVS_KEY_ATTR_NSH)
                return false;

        ovs_match_init(&match, &key, true, NULL);
        return !nsh_key_put_from_nlattr(nsh_key, &match, false, true, log);
}

/* Return false if there are any non-masked bits set.
 * Mask follows data immediately, before any netlink padding.
 */
static bool validate_masked(u8 *data, int len)
{
        u8 *mask = data + len;

        while (len--)
                if (*data++ & ~*mask++)
                        return false;

        return true;
}

static int validate_set(const struct nlattr *a,
                        const struct sw_flow_key *flow_key,
                        struct sw_flow_actions **sfa, bool *skip_copy,
                        u8 mac_proto, __be16 eth_type, bool masked, bool log)
{
        const struct nlattr *ovs_key = nla_data(a);
        int key_type = nla_type(ovs_key);
        size_t key_len;

        /* There can be only one key in a action */
        if (!nla_ok(ovs_key, nla_len(a)) ||
            nla_total_size(nla_len(ovs_key)) != nla_len(a))
                return -EINVAL;

        key_len = nla_len(ovs_key);
        if (masked)
                key_len /= 2;

        if (key_type > OVS_KEY_ATTR_MAX ||
            !check_attr_len(key_len, ovs_key_lens[key_type].len))
                return -EINVAL;

        if (masked && !validate_masked(nla_data(ovs_key), key_len))
                return -EINVAL;

        switch (key_type) {
        case OVS_KEY_ATTR_PRIORITY:
        case OVS_KEY_ATTR_SKB_MARK:
        case OVS_KEY_ATTR_CT_MARK:
        case OVS_KEY_ATTR_CT_LABELS:
                break;

        case OVS_KEY_ATTR_ETHERNET:
                if (mac_proto != MAC_PROTO_ETHERNET)
                        return -EINVAL;
                break;

        case OVS_KEY_ATTR_TUNNEL: {
                int err;

                if (masked)
                        return -EINVAL; /* Masked tunnel set not supported. */

                *skip_copy = true;
                err = validate_and_copy_set_tun(a, sfa, log);
                if (err)
                        return err;
                break;
        }
        case OVS_KEY_ATTR_IPV4: {
                const struct ovs_key_ipv4 *ipv4_key;

                if (eth_type != htons(ETH_P_IP))
                        return -EINVAL;

                ipv4_key = nla_data(ovs_key);

                if (masked) {
                        const struct ovs_key_ipv4 *mask = ipv4_key + 1;

                        /* Non-writeable fields. */
                        if (mask->ipv4_proto || mask->ipv4_frag)
                                return -EINVAL;
                } else {
                        if (ipv4_key->ipv4_proto != flow_key->ip.proto)
                                return -EINVAL;

                        if (ipv4_key->ipv4_frag != flow_key->ip.frag)
                                return -EINVAL;
                }
                break;
        }
        case OVS_KEY_ATTR_IPV6: {
                const struct ovs_key_ipv6 *ipv6_key;

                if (eth_type != htons(ETH_P_IPV6))
                        return -EINVAL;

                ipv6_key = nla_data(ovs_key);

                if (masked) {
                        const struct ovs_key_ipv6 *mask = ipv6_key + 1;

                        /* Non-writeable fields. */
                        if (mask->ipv6_proto || mask->ipv6_frag)
                                return -EINVAL;

                        /* Invalid bits in the flow label mask? */
                        if (ntohl(mask->ipv6_label) & 0xFFF00000)
                                return -EINVAL;
                } else {
                        if (ipv6_key->ipv6_proto != flow_key->ip.proto)
                                return -EINVAL;

                        if (ipv6_key->ipv6_frag != flow_key->ip.frag)
                                return -EINVAL;
                }
                if (ntohl(ipv6_key->ipv6_label) & 0xFFF00000)
                        return -EINVAL;

                break;
        }
        case OVS_KEY_ATTR_TCP:
                if ((eth_type != htons(ETH_P_IP) &&
                     eth_type != htons(ETH_P_IPV6)) ||
                    flow_key->ip.proto != IPPROTO_TCP)
                        return -EINVAL;

                break;

        case OVS_KEY_ATTR_UDP:
                if ((eth_type != htons(ETH_P_IP) &&
                     eth_type != htons(ETH_P_IPV6)) ||
                    flow_key->ip.proto != IPPROTO_UDP)
                        return -EINVAL;

                break;

        case OVS_KEY_ATTR_MPLS:
                if (!eth_p_mpls(eth_type))
                        return -EINVAL;
                if (key_len != sizeof(struct ovs_key_mpls))
                        return -EINVAL;
                break;

        case OVS_KEY_ATTR_SCTP:
                if ((eth_type != htons(ETH_P_IP) &&
                     eth_type != htons(ETH_P_IPV6)) ||
                    flow_key->ip.proto != IPPROTO_SCTP)
                        return -EINVAL;

                break;

        default:
                return -EINVAL;
        }

        /* Convert non-masked non-tunnel set actions to masked set actions. */
        if (!masked && key_type != OVS_KEY_ATTR_TUNNEL) {
                int start, len = key_len * 2;
                struct nlattr *at;

                *skip_copy = true;

                start = add_nested_action_start(sfa,
                                                OVS_ACTION_ATTR_SET_TO_MASKED,
                                                log);
                if (start < 0)
                        return start;

                at = __add_action(sfa, key_type, NULL, len, log);
                if (IS_ERR(at))
                        return PTR_ERR(at);

                memcpy(nla_data(at), nla_data(ovs_key), key_len); /* Key. */
                memset(nla_data(at) + key_len, 0xff, key_len);    /* Mask. */
                /* Clear non-writeable bits from otherwise writeable fields. */
                if (key_type == OVS_KEY_ATTR_IPV6) {
                        struct ovs_key_ipv6 *mask = nla_data(at) + key_len;

                        mask->ipv6_label &= htonl(0x000FFFFF);
                }
                add_nested_action_end(*sfa, start);
        }

        return 0;
}

static int validate_userspace(const struct nlattr *attr)
{
        static const struct nla_policy userspace_policy[OVS_USERSPACE_ATTR_MAX + 1] = {
                [OVS_USERSPACE_ATTR_PID] = {.type = NLA_U32 },
                [OVS_USERSPACE_ATTR_USERDATA] = {.type = NLA_UNSPEC },
                [OVS_USERSPACE_ATTR_EGRESS_TUN_PORT] = {.type = NLA_U32 },
        };
        struct nlattr *a[OVS_USERSPACE_ATTR_MAX + 1];
        int error;

        error = nla_parse_deprecated_strict(a, OVS_USERSPACE_ATTR_MAX,
                                            nla_data(attr), nla_len(attr),
                                            userspace_policy, NULL);
        if (error)
                return error;

        if (!a[OVS_USERSPACE_ATTR_PID] ||
            !nla_get_u32(a[OVS_USERSPACE_ATTR_PID]))
                return -EINVAL;

        return 0;
}

static const struct nla_policy cpl_policy[OVS_CHECK_PKT_LEN_ATTR_MAX + 1] = {
        [OVS_CHECK_PKT_LEN_ATTR_PKT_LEN] = {.type = NLA_U16 },
        [OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER] = {.type = NLA_NESTED },
        [OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL] = {.type = NLA_NESTED },
};

static int validate_and_copy_check_pkt_len(struct net *net,
                                           const struct nlattr *attr,
                                           const struct sw_flow_key *key,
                                           struct sw_flow_actions **sfa,
                                           __be16 eth_type, __be16 vlan_tci,
                                           u32 mpls_label_count,
                                           bool log, bool last, u32 depth)
{
        const struct nlattr *acts_if_greater, *acts_if_lesser_eq;
        struct nlattr *a[OVS_CHECK_PKT_LEN_ATTR_MAX + 1];
        struct check_pkt_len_arg arg;
        int nested_acts_start;
        int start, err;

        err = nla_parse_deprecated_strict(a, OVS_CHECK_PKT_LEN_ATTR_MAX,
                                          nla_data(attr), nla_len(attr),
                                          cpl_policy, NULL);
        if (err)
                return err;

        if (!a[OVS_CHECK_PKT_LEN_ATTR_PKT_LEN] ||
            !nla_get_u16(a[OVS_CHECK_PKT_LEN_ATTR_PKT_LEN]))
                return -EINVAL;

        acts_if_lesser_eq = a[OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL];
        acts_if_greater = a[OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER];

        /* Both the nested action should be present. */
        if (!acts_if_greater || !acts_if_lesser_eq)
                return -EINVAL;

        /* validation done, copy the nested actions. */
        start = add_nested_action_start(sfa, OVS_ACTION_ATTR_CHECK_PKT_LEN,
                                        log);
        if (start < 0)
                return start;

        arg.pkt_len = nla_get_u16(a[OVS_CHECK_PKT_LEN_ATTR_PKT_LEN]);
        arg.exec_for_lesser_equal =
                last || !actions_may_change_flow(acts_if_lesser_eq);
        arg.exec_for_greater =
                last || !actions_may_change_flow(acts_if_greater);

        err = ovs_nla_add_action(sfa, OVS_CHECK_PKT_LEN_ATTR_ARG, &arg,
                                 sizeof(arg), log);
        if (err)
                return err;

        nested_acts_start = add_nested_action_start(sfa,
                OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL, log);
        if (nested_acts_start < 0)
                return nested_acts_start;

        err = __ovs_nla_copy_actions(net, acts_if_lesser_eq, key, sfa,
                                     eth_type, vlan_tci, mpls_label_count, log,
                                     depth + 1);

        if (err)
                return err;

        add_nested_action_end(*sfa, nested_acts_start);

        nested_acts_start = add_nested_action_start(sfa,
                OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER, log);
        if (nested_acts_start < 0)
                return nested_acts_start;

        err = __ovs_nla_copy_actions(net, acts_if_greater, key, sfa,
                                     eth_type, vlan_tci, mpls_label_count, log,
                                     depth + 1);

        if (err)
                return err;

        add_nested_action_end(*sfa, nested_acts_start);
        add_nested_action_end(*sfa, start);
        return 0;
}

static int validate_psample(const struct nlattr *attr)
{
        static const struct nla_policy policy[OVS_PSAMPLE_ATTR_MAX + 1] = {
                [OVS_PSAMPLE_ATTR_GROUP] = { .type = NLA_U32 },
                [OVS_PSAMPLE_ATTR_COOKIE] = {
                        .type = NLA_BINARY,
                        .len = OVS_PSAMPLE_COOKIE_MAX_SIZE,
                },
        };
        struct nlattr *a[OVS_PSAMPLE_ATTR_MAX + 1];
        int err;

        if (!IS_ENABLED(CONFIG_PSAMPLE))
                return -EOPNOTSUPP;

        err = nla_parse_nested(a, OVS_PSAMPLE_ATTR_MAX, attr, policy, NULL);
        if (err)
                return err;

        return a[OVS_PSAMPLE_ATTR_GROUP] ? 0 : -EINVAL;
}

static int copy_action(const struct nlattr *from,
                       struct sw_flow_actions **sfa, bool log)
{
        int totlen = NLA_ALIGN(from->nla_len);
        struct nlattr *to;

        to = reserve_sfa_size(sfa, from->nla_len, log);
        if (IS_ERR(to))
                return PTR_ERR(to);

        memcpy(to, from, totlen);
        return 0;
}

static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
                                  const struct sw_flow_key *key,
                                  struct sw_flow_actions **sfa,
                                  __be16 eth_type, __be16 vlan_tci,
                                  u32 mpls_label_count, bool log,
                                  u32 depth)
{
        u8 mac_proto = ovs_key_mac_proto(key);
        const struct nlattr *a;
        int rem, err;

        if (depth > OVS_COPY_ACTIONS_MAX_DEPTH)
                return -EOVERFLOW;

        nla_for_each_nested(a, attr, rem) {
                /* Expected argument lengths, (u32)-1 for variable length. */
                static const u32 action_lens[OVS_ACTION_ATTR_MAX + 1] = {
                        [OVS_ACTION_ATTR_OUTPUT] = sizeof(u32),
                        [OVS_ACTION_ATTR_RECIRC] = sizeof(u32),
                        [OVS_ACTION_ATTR_USERSPACE] = (u32)-1,
                        [OVS_ACTION_ATTR_PUSH_MPLS] = sizeof(struct ovs_action_push_mpls),
                        [OVS_ACTION_ATTR_POP_MPLS] = sizeof(__be16),
                        [OVS_ACTION_ATTR_PUSH_VLAN] = sizeof(struct ovs_action_push_vlan),
                        [OVS_ACTION_ATTR_POP_VLAN] = 0,
                        [OVS_ACTION_ATTR_SET] = (u32)-1,
                        [OVS_ACTION_ATTR_SET_MASKED] = (u32)-1,
                        [OVS_ACTION_ATTR_SAMPLE] = (u32)-1,
                        [OVS_ACTION_ATTR_HASH] = sizeof(struct ovs_action_hash),
                        [OVS_ACTION_ATTR_CT] = (u32)-1,
                        [OVS_ACTION_ATTR_CT_CLEAR] = 0,
                        [OVS_ACTION_ATTR_TRUNC] = sizeof(struct ovs_action_trunc),
                        [OVS_ACTION_ATTR_PUSH_ETH] = sizeof(struct ovs_action_push_eth),
                        [OVS_ACTION_ATTR_POP_ETH] = 0,
                        [OVS_ACTION_ATTR_PUSH_NSH] = (u32)-1,
                        [OVS_ACTION_ATTR_POP_NSH] = 0,
                        [OVS_ACTION_ATTR_METER] = sizeof(u32),
                        [OVS_ACTION_ATTR_CLONE] = (u32)-1,
                        [OVS_ACTION_ATTR_CHECK_PKT_LEN] = (u32)-1,
                        [OVS_ACTION_ATTR_ADD_MPLS] = sizeof(struct ovs_action_add_mpls),
                        [OVS_ACTION_ATTR_DEC_TTL] = (u32)-1,
                        [OVS_ACTION_ATTR_DROP] = sizeof(u32),
                        [OVS_ACTION_ATTR_PSAMPLE] = (u32)-1,
                };
                const struct ovs_action_push_vlan *vlan;
                int type = nla_type(a);
                bool skip_copy;

                if (type > OVS_ACTION_ATTR_MAX ||
                    (action_lens[type] != nla_len(a) &&
                     action_lens[type] != (u32)-1))
                        return -EINVAL;

                skip_copy = false;
                switch (type) {
                case OVS_ACTION_ATTR_UNSPEC:
                        return -EINVAL;

                case OVS_ACTION_ATTR_USERSPACE:
                        err = validate_userspace(a);
                        if (err)
                                return err;
                        break;

                case OVS_ACTION_ATTR_OUTPUT:
                        if (nla_get_u32(a) >= DP_MAX_PORTS)
                                return -EINVAL;
                        break;

                case OVS_ACTION_ATTR_TRUNC: {
                        const struct ovs_action_trunc *trunc = nla_data(a);

                        if (trunc->max_len < ETH_HLEN)
                                return -EINVAL;
                        break;
                }

                case OVS_ACTION_ATTR_HASH: {
                        const struct ovs_action_hash *act_hash = nla_data(a);

                        switch (act_hash->hash_alg) {
                        case OVS_HASH_ALG_L4:
                                fallthrough;
                        case OVS_HASH_ALG_SYM_L4:
                                break;
                        default:
                                return  -EINVAL;
                        }

                        break;
                }

                case OVS_ACTION_ATTR_POP_VLAN:
                        if (mac_proto != MAC_PROTO_ETHERNET)
                                return -EINVAL;
                        vlan_tci = htons(0);
                        break;

                case OVS_ACTION_ATTR_PUSH_VLAN:
                        if (mac_proto != MAC_PROTO_ETHERNET)
                                return -EINVAL;
                        vlan = nla_data(a);
                        if (!eth_type_vlan(vlan->vlan_tpid))
                                return -EINVAL;
                        if (!(vlan->vlan_tci & htons(VLAN_CFI_MASK)))
                                return -EINVAL;
                        vlan_tci = vlan->vlan_tci;
                        break;

                case OVS_ACTION_ATTR_RECIRC:
                        break;

                case OVS_ACTION_ATTR_ADD_MPLS: {
                        const struct ovs_action_add_mpls *mpls = nla_data(a);

                        if (!eth_p_mpls(mpls->mpls_ethertype))
                                return -EINVAL;

                        if (mpls->tun_flags & OVS_MPLS_L3_TUNNEL_FLAG_MASK) {
                                if (vlan_tci & htons(VLAN_CFI_MASK) ||
                                    (eth_type != htons(ETH_P_IP) &&
                                     eth_type != htons(ETH_P_IPV6) &&
                                     eth_type != htons(ETH_P_ARP) &&
                                     eth_type != htons(ETH_P_RARP) &&
                                     !eth_p_mpls(eth_type)))
                                        return -EINVAL;
                                mpls_label_count++;
                        } else {
                                if (mac_proto == MAC_PROTO_ETHERNET) {
                                        mpls_label_count = 1;
                                        mac_proto = MAC_PROTO_NONE;
                                } else {
                                        mpls_label_count++;
                                }
                        }
                        eth_type = mpls->mpls_ethertype;
                        break;
                }

                case OVS_ACTION_ATTR_PUSH_MPLS: {
                        const struct ovs_action_push_mpls *mpls = nla_data(a);

                        if (!eth_p_mpls(mpls->mpls_ethertype))
                                return -EINVAL;
                        /* Prohibit push MPLS other than to a white list
                         * for packets that have a known tag order.
                         */
                        if (vlan_tci & htons(VLAN_CFI_MASK) ||
                            (eth_type != htons(ETH_P_IP) &&
                             eth_type != htons(ETH_P_IPV6) &&
                             eth_type != htons(ETH_P_ARP) &&
                             eth_type != htons(ETH_P_RARP) &&
                             !eth_p_mpls(eth_type)))
                                return -EINVAL;
                        eth_type = mpls->mpls_ethertype;
                        mpls_label_count++;
                        break;
                }

                case OVS_ACTION_ATTR_POP_MPLS: {
                        __be16  proto;
                        if (vlan_tci & htons(VLAN_CFI_MASK) ||
                            !eth_p_mpls(eth_type))
                                return -EINVAL;

                        /* Disallow subsequent L2.5+ set actions and mpls_pop
                         * actions once the last MPLS label in the packet is
                         * popped as there is no check here to ensure that
                         * the new eth type is valid and thus set actions could
                         * write off the end of the packet or otherwise corrupt
                         * it.
                         *
                         * Support for these actions is planned using packet
                         * recirculation.
                         */
                        proto = nla_get_be16(a);

                        if (proto == htons(ETH_P_TEB) &&
                            mac_proto != MAC_PROTO_NONE)
                                return -EINVAL;

                        mpls_label_count--;

                        if (!eth_p_mpls(proto) || !mpls_label_count)
                                eth_type = htons(0);
                        else
                                eth_type =  proto;

                        break;
                }

                case OVS_ACTION_ATTR_SET:
                        err = validate_set(a, key, sfa,
                                           &skip_copy, mac_proto, eth_type,
                                           false, log);
                        if (err)
                                return err;
                        break;

                case OVS_ACTION_ATTR_SET_MASKED:
                        err = validate_set(a, key, sfa,
                                           &skip_copy, mac_proto, eth_type,
                                           true, log);
                        if (err)
                                return err;
                        break;

                case OVS_ACTION_ATTR_SAMPLE: {
                        bool last = nla_is_last(a, rem);

                        err = validate_and_copy_sample(net, a, key, sfa,
                                                       eth_type, vlan_tci,
                                                       mpls_label_count,
                                                       log, last, depth);
                        if (err)
                                return err;
                        skip_copy = true;
                        break;
                }

                case OVS_ACTION_ATTR_CT:
                        err = ovs_ct_copy_action(net, a, key, sfa, log);
                        if (err)
                                return err;
                        skip_copy = true;
                        break;

                case OVS_ACTION_ATTR_CT_CLEAR:
                        break;

                case OVS_ACTION_ATTR_PUSH_ETH:
                        /* Disallow pushing an Ethernet header if one
                         * is already present */
                        if (mac_proto != MAC_PROTO_NONE)
                                return -EINVAL;
                        mac_proto = MAC_PROTO_ETHERNET;
                        break;

                case OVS_ACTION_ATTR_POP_ETH:
                        if (mac_proto != MAC_PROTO_ETHERNET)
                                return -EINVAL;
                        if (vlan_tci & htons(VLAN_CFI_MASK))
                                return -EINVAL;
                        mac_proto = MAC_PROTO_NONE;
                        break;

                case OVS_ACTION_ATTR_PUSH_NSH:
                        if (mac_proto != MAC_PROTO_ETHERNET) {
                                u8 next_proto;

                                next_proto = tun_p_from_eth_p(eth_type);
                                if (!next_proto)
                                        return -EINVAL;
                        }
                        mac_proto = MAC_PROTO_NONE;
                        if (!validate_push_nsh(a, log))
                                return -EINVAL;
                        break;

                case OVS_ACTION_ATTR_POP_NSH: {
                        __be16 inner_proto;

                        if (eth_type != htons(ETH_P_NSH))
                                return -EINVAL;
                        inner_proto = tun_p_to_eth_p(key->nsh.base.np);
                        if (!inner_proto)
                                return -EINVAL;
                        if (key->nsh.base.np == TUN_P_ETHERNET)
                                mac_proto = MAC_PROTO_ETHERNET;
                        else
                                mac_proto = MAC_PROTO_NONE;
                        break;
                }

                case OVS_ACTION_ATTR_METER:
                        /* Non-existent meters are simply ignored.  */
                        break;

                case OVS_ACTION_ATTR_CLONE: {
                        bool last = nla_is_last(a, rem);

                        err = validate_and_copy_clone(net, a, key, sfa,
                                                      eth_type, vlan_tci,
                                                      mpls_label_count,
                                                      log, last, depth);
                        if (err)
                                return err;
                        skip_copy = true;
                        break;
                }

                case OVS_ACTION_ATTR_CHECK_PKT_LEN: {
                        bool last = nla_is_last(a, rem);

                        err = validate_and_copy_check_pkt_len(net, a, key, sfa,
                                                              eth_type,
                                                              vlan_tci,
                                                              mpls_label_count,
                                                              log, last,
                                                              depth);
                        if (err)
                                return err;
                        skip_copy = true;
                        break;
                }

                case OVS_ACTION_ATTR_DEC_TTL:
                        err = validate_and_copy_dec_ttl(net, a, key, sfa,
                                                        eth_type, vlan_tci,
                                                        mpls_label_count, log,
                                                        depth);
                        if (err)
                                return err;
                        skip_copy = true;
                        break;

                case OVS_ACTION_ATTR_DROP:
                        if (!nla_is_last(a, rem))
                                return -EINVAL;
                        break;

                case OVS_ACTION_ATTR_PSAMPLE:
                        err = validate_psample(a);
                        if (err)
                                return err;
                        break;

                default:
                        OVS_NLERR(log, "Unknown Action type %d", type);
                        return -EINVAL;
                }
                if (!skip_copy) {
                        err = copy_action(a, sfa, log);
                        if (err)
                                return err;
                }
        }

        if (rem > 0)
                return -EINVAL;

        return 0;
}

/* 'key' must be the masked key. */
int ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
                         const struct sw_flow_key *key,
                         struct sw_flow_actions **sfa, bool log)
{
        int err;
        u32 mpls_label_count = 0;

        *sfa = nla_alloc_flow_actions(nla_len(attr));
        if (IS_ERR(*sfa))
                return PTR_ERR(*sfa);

        if (eth_p_mpls(key->eth.type))
                mpls_label_count = hweight_long(key->mpls.num_labels_mask);

        (*sfa)->orig_len = nla_len(attr);
        err = __ovs_nla_copy_actions(net, attr, key, sfa, key->eth.type,
                                     key->eth.vlan.tci, mpls_label_count, log,
                                     0);
        if (err)
                ovs_nla_free_flow_actions(*sfa);

        return err;
}

static int sample_action_to_attr(const struct nlattr *attr,
                                 struct sk_buff *skb)
{
        struct nlattr *start, *ac_start = NULL, *sample_arg;
        int err = 0, rem = nla_len(attr);
        const struct sample_arg *arg;
        struct nlattr *actions;

        start = nla_nest_start_noflag(skb, OVS_ACTION_ATTR_SAMPLE);
        if (!start)
                return -EMSGSIZE;

        sample_arg = nla_data(attr);
        arg = nla_data(sample_arg);
        actions = nla_next(sample_arg, &rem);

        if (nla_put_u32(skb, OVS_SAMPLE_ATTR_PROBABILITY, arg->probability)) {
                err = -EMSGSIZE;
                goto out;
        }

        ac_start = nla_nest_start_noflag(skb, OVS_SAMPLE_ATTR_ACTIONS);
        if (!ac_start) {
                err = -EMSGSIZE;
                goto out;
        }

        err = ovs_nla_put_actions(actions, rem, skb);

out:
        if (err) {
                nla_nest_cancel(skb, ac_start);
                nla_nest_cancel(skb, start);
        } else {
                nla_nest_end(skb, ac_start);
                nla_nest_end(skb, start);
        }

        return err;
}

static int clone_action_to_attr(const struct nlattr *attr,
                                struct sk_buff *skb)
{
        struct nlattr *start;
        int err = 0, rem = nla_len(attr);

        start = nla_nest_start_noflag(skb, OVS_ACTION_ATTR_CLONE);
        if (!start)
                return -EMSGSIZE;

        /* Skipping the OVS_CLONE_ATTR_EXEC that is always the first attribute. */
        attr = nla_next(nla_data(attr), &rem);
        err = ovs_nla_put_actions(attr, rem, skb);

        if (err)
                nla_nest_cancel(skb, start);
        else
                nla_nest_end(skb, start);

        return err;
}

static int check_pkt_len_action_to_attr(const struct nlattr *attr,
                                        struct sk_buff *skb)
{
        struct nlattr *start, *ac_start = NULL;
        const struct check_pkt_len_arg *arg;
        const struct nlattr *a, *cpl_arg;
        int err = 0, rem = nla_len(attr);

        start = nla_nest_start_noflag(skb, OVS_ACTION_ATTR_CHECK_PKT_LEN);
        if (!start)
                return -EMSGSIZE;

        /* The first nested attribute in 'attr' is always
         * 'OVS_CHECK_PKT_LEN_ATTR_ARG'.
         */
        cpl_arg = nla_data(attr);
        arg = nla_data(cpl_arg);

        if (nla_put_u16(skb, OVS_CHECK_PKT_LEN_ATTR_PKT_LEN, arg->pkt_len)) {
                err = -EMSGSIZE;
                goto out;
        }

        /* Second nested attribute in 'attr' is always
         * 'OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL'.
         */
        a = nla_next(cpl_arg, &rem);
        ac_start =  nla_nest_start_noflag(skb,
                                          OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL);
        if (!ac_start) {
                err = -EMSGSIZE;
                goto out;
        }

        err = ovs_nla_put_actions(nla_data(a), nla_len(a), skb);
        if (err) {
                nla_nest_cancel(skb, ac_start);
                goto out;
        } else {
                nla_nest_end(skb, ac_start);
        }

        /* Third nested attribute in 'attr' is always
         * OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER.
         */
        a = nla_next(a, &rem);
        ac_start =  nla_nest_start_noflag(skb,
                                          OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER);
        if (!ac_start) {
                err = -EMSGSIZE;
                goto out;
        }

        err = ovs_nla_put_actions(nla_data(a), nla_len(a), skb);
        if (err) {
                nla_nest_cancel(skb, ac_start);
                goto out;
        } else {
                nla_nest_end(skb, ac_start);
        }

        nla_nest_end(skb, start);
        return 0;

out:
        nla_nest_cancel(skb, start);
        return err;
}

static int dec_ttl_action_to_attr(const struct nlattr *attr,
                                  struct sk_buff *skb)
{
        struct nlattr *start, *action_start;
        const struct nlattr *a;
        int err = 0, rem;

        start = nla_nest_start_noflag(skb, OVS_ACTION_ATTR_DEC_TTL);
        if (!start)
                return -EMSGSIZE;

        nla_for_each_attr(a, nla_data(attr), nla_len(attr), rem) {
                switch (nla_type(a)) {
                case OVS_DEC_TTL_ATTR_ACTION:

                        action_start = nla_nest_start_noflag(skb, OVS_DEC_TTL_ATTR_ACTION);
                        if (!action_start) {
                                err = -EMSGSIZE;
                                goto out;
                        }

                        err = ovs_nla_put_actions(nla_data(a), nla_len(a), skb);
                        if (err)
                                goto out;

                        nla_nest_end(skb, action_start);
                        break;

                default:
                        /* Ignore all other option to be future compatible */
                        break;
                }
        }

        nla_nest_end(skb, start);
        return 0;

out:
        nla_nest_cancel(skb, start);
        return err;
}

static int set_action_to_attr(const struct nlattr *a, struct sk_buff *skb)
{
        const struct nlattr *ovs_key = nla_data(a);
        int key_type = nla_type(ovs_key);
        struct nlattr *start;
        int err;

        switch (key_type) {
        case OVS_KEY_ATTR_TUNNEL_INFO: {
                struct ovs_tunnel_info *ovs_tun = nla_data(ovs_key);
                struct ip_tunnel_info *tun_info = &ovs_tun->tun_dst->u.tun_info;

                start = nla_nest_start_noflag(skb, OVS_ACTION_ATTR_SET);
                if (!start)
                        return -EMSGSIZE;

                err =  ip_tun_to_nlattr(skb, &tun_info->key,
                                        ip_tunnel_info_opts(tun_info),
                                        tun_info->options_len,
                                        ip_tunnel_info_af(tun_info), tun_info->mode);
                if (err)
                        return err;
                nla_nest_end(skb, start);
                break;
        }
        default:
                if (nla_put(skb, OVS_ACTION_ATTR_SET, nla_len(a), ovs_key))
                        return -EMSGSIZE;
                break;
        }

        return 0;
}

static int masked_set_action_to_set_action_attr(const struct nlattr *a,
                                                struct sk_buff *skb)
{
        const struct nlattr *ovs_key = nla_data(a);
        struct nlattr *nla;
        size_t key_len = nla_len(ovs_key) / 2;

        /* Revert the conversion we did from a non-masked set action to
         * masked set action.
         */
        nla = nla_nest_start_noflag(skb, OVS_ACTION_ATTR_SET);
        if (!nla)
                return -EMSGSIZE;

        if (nla_put(skb, nla_type(ovs_key), key_len, nla_data(ovs_key)))
                return -EMSGSIZE;

        nla_nest_end(skb, nla);
        return 0;
}

int ovs_nla_put_actions(const struct nlattr *attr, int len, struct sk_buff *skb)
{
        const struct nlattr *a;
        int rem, err;

        nla_for_each_attr(a, attr, len, rem) {
                int type = nla_type(a);

                switch (type) {
                case OVS_ACTION_ATTR_SET:
                        err = set_action_to_attr(a, skb);
                        if (err)
                                return err;
                        break;

                case OVS_ACTION_ATTR_SET_TO_MASKED:
                        err = masked_set_action_to_set_action_attr(a, skb);
                        if (err)
                                return err;
                        break;

                case OVS_ACTION_ATTR_SAMPLE:
                        err = sample_action_to_attr(a, skb);
                        if (err)
                                return err;
                        break;

                case OVS_ACTION_ATTR_CT:
                        err = ovs_ct_action_to_attr(nla_data(a), skb);
                        if (err)
                                return err;
                        break;

                case OVS_ACTION_ATTR_CLONE:
                        err = clone_action_to_attr(a, skb);
                        if (err)
                                return err;
                        break;

                case OVS_ACTION_ATTR_CHECK_PKT_LEN:
                        err = check_pkt_len_action_to_attr(a, skb);
                        if (err)
                                return err;
                        break;

                case OVS_ACTION_ATTR_DEC_TTL:
                        err = dec_ttl_action_to_attr(a, skb);
                        if (err)
                                return err;
                        break;

                default:
                        if (nla_put(skb, type, nla_len(a), nla_data(a)))
                                return -EMSGSIZE;
                        break;
                }
        }

        return 0;
}
























































































































































































































































































































 1448 



 1448 
 1241 
 1282 









  203 




   27 



   24 
  209 













   29 


































  571 























































  694 




  698 








   41 
  657 


















  410 
  236 
  300 
    9 




   11 
  490 



























  603 








  601 



















  603 











    1 
  736 

  603 






















































   10 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _NET_NEIGHBOUR_H
#define _NET_NEIGHBOUR_H

#include <linux/neighbour.h>

/*
 *        Generic neighbour manipulation
 *
 *        Authors:
 *        Pedro Roque                <roque@di.fc.ul.pt>
 *        Alexey Kuznetsov        <kuznet@ms2.inr.ac.ru>
 *
 *         Changes:
 *
 *        Harald Welte:                <laforge@gnumonks.org>
 *                - Add neighbour cache statistics like rtstat
 */

#include <linux/atomic.h>
#include <linux/refcount.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/rcupdate.h>
#include <linux/seq_file.h>
#include <linux/bitmap.h>

#include <linux/err.h>
#include <linux/sysctl.h>
#include <linux/workqueue.h>
#include <net/rtnetlink.h>
#include <net/neighbour_tables.h>

/*
 * NUD stands for "neighbor unreachability detection"
 */

#define NUD_IN_TIMER        (NUD_INCOMPLETE|NUD_REACHABLE|NUD_DELAY|NUD_PROBE)
#define NUD_VALID        (NUD_PERMANENT|NUD_NOARP|NUD_REACHABLE|NUD_PROBE|NUD_STALE|NUD_DELAY)
#define NUD_CONNECTED        (NUD_PERMANENT|NUD_NOARP|NUD_REACHABLE)

struct neighbour;

enum {
        NEIGH_VAR_MCAST_PROBES,
        NEIGH_VAR_UCAST_PROBES,
        NEIGH_VAR_APP_PROBES,
        NEIGH_VAR_MCAST_REPROBES,
        NEIGH_VAR_RETRANS_TIME,
        NEIGH_VAR_BASE_REACHABLE_TIME,
        NEIGH_VAR_DELAY_PROBE_TIME,
        NEIGH_VAR_INTERVAL_PROBE_TIME_MS,
        NEIGH_VAR_GC_STALETIME,
        NEIGH_VAR_QUEUE_LEN_BYTES,
        NEIGH_VAR_PROXY_QLEN,
        NEIGH_VAR_ANYCAST_DELAY,
        NEIGH_VAR_PROXY_DELAY,
        NEIGH_VAR_LOCKTIME,
#define NEIGH_VAR_DATA_MAX (NEIGH_VAR_LOCKTIME + 1)
        /* Following are used as a second way to access one of the above */
        NEIGH_VAR_QUEUE_LEN, /* same data as NEIGH_VAR_QUEUE_LEN_BYTES */
        NEIGH_VAR_RETRANS_TIME_MS, /* same data as NEIGH_VAR_RETRANS_TIME */
        NEIGH_VAR_BASE_REACHABLE_TIME_MS, /* same data as NEIGH_VAR_BASE_REACHABLE_TIME */
        /* Following are used by "default" only */
        NEIGH_VAR_GC_INTERVAL,
        NEIGH_VAR_GC_THRESH1,
        NEIGH_VAR_GC_THRESH2,
        NEIGH_VAR_GC_THRESH3,
        NEIGH_VAR_MAX
};

struct neigh_parms {
        possible_net_t net;
        struct net_device *dev;
        netdevice_tracker dev_tracker;
        struct list_head list;
        int        (*neigh_setup)(struct neighbour *);
        struct neigh_table *tbl;

        void        *sysctl_table;

        int dead;
        refcount_t refcnt;
        struct rcu_head rcu_head;

        int        reachable_time;
        u32        qlen;
        int        data[NEIGH_VAR_DATA_MAX];
        DECLARE_BITMAP(data_state, NEIGH_VAR_DATA_MAX);
};

static inline void neigh_var_set(struct neigh_parms *p, int index, int val)
{
        set_bit(index, p->data_state);
        WRITE_ONCE(p->data[index], val);
}

#define __NEIGH_VAR(p, attr) ((p)->data[NEIGH_VAR_ ## attr])
#define NEIGH_VAR(p, attr) READ_ONCE(__NEIGH_VAR(p, attr))
#define NEIGH_VAR_PTR(p, attr) (&(__NEIGH_VAR(p, attr)))

/* In ndo_neigh_setup, NEIGH_VAR_INIT should be used.
 * In other cases, NEIGH_VAR_SET should be used.
 */
#define NEIGH_VAR_INIT(p, attr, val) (__NEIGH_VAR(p, attr) = val)
#define NEIGH_VAR_SET(p, attr, val) neigh_var_set(p, NEIGH_VAR_ ## attr, val)

static inline void neigh_parms_data_state_setall(struct neigh_parms *p)
{
        bitmap_fill(p->data_state, NEIGH_VAR_DATA_MAX);
}

static inline void neigh_parms_data_state_cleanall(struct neigh_parms *p)
{
        bitmap_zero(p->data_state, NEIGH_VAR_DATA_MAX);
}

struct neigh_statistics {
        unsigned long allocs;                /* number of allocated neighs */
        unsigned long destroys;                /* number of destroyed neighs */
        unsigned long hash_grows;        /* number of hash resizes */

        unsigned long res_failed;        /* number of failed resolutions */

        unsigned long lookups;                /* number of lookups */
        unsigned long hits;                /* number of hits (among lookups) */

        unsigned long rcv_probes_mcast;        /* number of received mcast ipv6 */
        unsigned long rcv_probes_ucast; /* number of received ucast ipv6 */

        unsigned long periodic_gc_runs;        /* number of periodic GC runs */
        unsigned long forced_gc_runs;        /* number of forced GC runs */

        unsigned long unres_discards;        /* number of unresolved drops */
        unsigned long table_fulls;      /* times even gc couldn't help */
};

#define NEIGH_CACHE_STAT_INC(tbl, field) this_cpu_inc((tbl)->stats->field)

struct neighbour {
        struct hlist_node        hash;
        struct hlist_node        dev_list;
        struct neigh_table        *tbl;
        struct neigh_parms        *parms;
        unsigned long                confirmed;
        unsigned long                updated;
        rwlock_t                lock;
        refcount_t                refcnt;
        unsigned int                arp_queue_len_bytes;
        struct sk_buff_head        arp_queue;
        struct timer_list        timer;
        unsigned long                used;
        atomic_t                probes;
        u8                        nud_state;
        u8                        type;
        u8                        dead;
        u8                        protocol;
        u32                        flags;
        seqlock_t                ha_lock;
        unsigned char                ha[ALIGN(MAX_ADDR_LEN, sizeof(unsigned long))] __aligned(8);
        struct hh_cache                hh;
        int                        (*output)(struct neighbour *, struct sk_buff *);
        const struct neigh_ops        *ops;
        struct list_head        gc_list;
        struct list_head        managed_list;
        struct rcu_head                rcu;
        struct net_device        *dev;
        netdevice_tracker        dev_tracker;
        u8                        primary_key[];
} __randomize_layout;

struct neigh_ops {
        int                        family;
        void                        (*solicit)(struct neighbour *, struct sk_buff *);
        void                        (*error_report)(struct neighbour *, struct sk_buff *);
        int                        (*output)(struct neighbour *, struct sk_buff *);
        int                        (*connected_output)(struct neighbour *, struct sk_buff *);
};

struct pneigh_entry {
        struct pneigh_entry        __rcu *next;
        possible_net_t                net;
        struct net_device        *dev;
        netdevice_tracker        dev_tracker;
        union {
                struct list_head        free_node;
                struct rcu_head                rcu;
        };
        u32                        flags;
        u8                        protocol;
        bool                        permanent;
        u32                        key[];
};

/*
 *        neighbour table manipulation
 */

#define NEIGH_NUM_HASH_RND        4

struct neigh_hash_table {
        struct hlist_head        *hash_heads;
        unsigned int                hash_shift;
        __u32                        hash_rnd[NEIGH_NUM_HASH_RND];
        struct rcu_head                rcu;
};


struct neigh_table {
        int                        family;
        unsigned int                entry_size;
        unsigned int                key_len;
        __be16                        protocol;
        __u32                        (*hash)(const void *pkey,
                                        const struct net_device *dev,
                                        __u32 *hash_rnd);
        bool                        (*key_eq)(const struct neighbour *, const void *pkey);
        int                        (*constructor)(struct neighbour *);
        int                        (*pconstructor)(struct pneigh_entry *);
        void                        (*pdestructor)(struct pneigh_entry *);
        void                        (*proxy_redo)(struct sk_buff *skb);
        int                        (*is_multicast)(const void *pkey);
        bool                        (*allow_add)(const struct net_device *dev,
                                             struct netlink_ext_ack *extack);
        char                        *id;
        struct neigh_parms        parms;
        struct list_head        parms_list;
        int                        gc_interval;
        int                        gc_thresh1;
        int                        gc_thresh2;
        int                        gc_thresh3;
        unsigned long                last_flush;
        struct delayed_work        gc_work;
        struct delayed_work        managed_work;
        struct timer_list         proxy_timer;
        struct sk_buff_head        proxy_queue;
        atomic_t                entries;
        atomic_t                gc_entries;
        struct list_head        gc_list;
        struct list_head        managed_list;
        spinlock_t                lock;
        unsigned long                last_rand;
        struct neigh_statistics        __percpu *stats;
        struct neigh_hash_table __rcu *nht;
        struct mutex                phash_lock;
        struct pneigh_entry        __rcu **phash_buckets;
};

static inline int neigh_parms_family(struct neigh_parms *p)
{
        return p->tbl->family;
}

#define NEIGH_PRIV_ALIGN        sizeof(long long)
#define NEIGH_ENTRY_SIZE(size)        ALIGN((size), NEIGH_PRIV_ALIGN)

static inline void *neighbour_priv(const struct neighbour *n)
{
        return (char *)n + n->tbl->entry_size;
}

/* flags for neigh_update() */
#define NEIGH_UPDATE_F_OVERRIDE                        BIT(0)
#define NEIGH_UPDATE_F_WEAK_OVERRIDE                BIT(1)
#define NEIGH_UPDATE_F_OVERRIDE_ISROUTER        BIT(2)
#define NEIGH_UPDATE_F_USE                        BIT(3)
#define NEIGH_UPDATE_F_MANAGED                        BIT(4)
#define NEIGH_UPDATE_F_EXT_LEARNED                BIT(5)
#define NEIGH_UPDATE_F_ISROUTER                        BIT(6)
#define NEIGH_UPDATE_F_ADMIN                        BIT(7)
#define NEIGH_UPDATE_F_EXT_VALIDATED                BIT(8)

/* In-kernel representation for NDA_FLAGS_EXT flags: */
#define NTF_OLD_MASK                0xff
#define NTF_EXT_SHIFT                8
#define NTF_EXT_MASK                (NTF_EXT_MANAGED | NTF_EXT_EXT_VALIDATED)

#define NTF_MANAGED                (NTF_EXT_MANAGED << NTF_EXT_SHIFT)
#define NTF_EXT_VALIDATED        (NTF_EXT_EXT_VALIDATED << NTF_EXT_SHIFT)

extern const struct nla_policy nda_policy[];

#define neigh_for_each_in_bucket(pos, head) hlist_for_each_entry(pos, head, hash)
#define neigh_for_each_in_bucket_rcu(pos, head) \
        hlist_for_each_entry_rcu(pos, head, hash)
#define neigh_for_each_in_bucket_safe(pos, tmp, head) \
        hlist_for_each_entry_safe(pos, tmp, head, hash)

static inline bool neigh_key_eq32(const struct neighbour *n, const void *pkey)
{
        return *(const u32 *)n->primary_key == *(const u32 *)pkey;
}

static inline bool neigh_key_eq128(const struct neighbour *n, const void *pkey)
{
        const u32 *n32 = (const u32 *)n->primary_key;
        const u32 *p32 = pkey;

        return ((n32[0] ^ p32[0]) | (n32[1] ^ p32[1]) |
                (n32[2] ^ p32[2]) | (n32[3] ^ p32[3])) == 0;
}

static inline struct neighbour *___neigh_lookup_noref(
        struct neigh_table *tbl,
        bool (*key_eq)(const struct neighbour *n, const void *pkey),
        __u32 (*hash)(const void *pkey,
                      const struct net_device *dev,
                      __u32 *hash_rnd),
        const void *pkey,
        struct net_device *dev)
{
        struct neigh_hash_table *nht = rcu_dereference(tbl->nht);
        struct neighbour *n;
        u32 hash_val;

        hash_val = hash(pkey, dev, nht->hash_rnd) >> (32 - nht->hash_shift);
        neigh_for_each_in_bucket_rcu(n, &nht->hash_heads[hash_val])
                if (n->dev == dev && key_eq(n, pkey))
                        return n;

        return NULL;
}

static inline struct neighbour *__neigh_lookup_noref(struct neigh_table *tbl,
                                                     const void *pkey,
                                                     struct net_device *dev)
{
        return ___neigh_lookup_noref(tbl, tbl->key_eq, tbl->hash, pkey, dev);
}

static inline void neigh_confirm(struct neighbour *n)
{
        if (n) {
                unsigned long now = jiffies;

                /* avoid dirtying neighbour */
                if (READ_ONCE(n->confirmed) != now)
                        WRITE_ONCE(n->confirmed, now);
        }
}

void neigh_table_init(int index, struct neigh_table *tbl);
int neigh_table_clear(int index, struct neigh_table *tbl);
struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey,
                               struct net_device *dev);
struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,
                                 struct net_device *dev, bool want_ref);
static inline struct neighbour *neigh_create(struct neigh_table *tbl,
                                             const void *pkey,
                                             struct net_device *dev)
{
        return __neigh_create(tbl, pkey, dev, true);
}
void neigh_destroy(struct neighbour *neigh);
int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb,
                       const bool immediate_ok);
int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new, u32 flags,
                 u32 nlmsg_pid);
void __neigh_set_probe_once(struct neighbour *neigh);
bool neigh_remove_one(struct neighbour *ndel);
void neigh_changeaddr(struct neigh_table *tbl, struct net_device *dev);
int neigh_ifdown(struct neigh_table *tbl, struct net_device *dev);
int neigh_carrier_down(struct neigh_table *tbl, struct net_device *dev);
int neigh_resolve_output(struct neighbour *neigh, struct sk_buff *skb);
int neigh_connected_output(struct neighbour *neigh, struct sk_buff *skb);
int neigh_direct_output(struct neighbour *neigh, struct sk_buff *skb);
struct neighbour *neigh_event_ns(struct neigh_table *tbl,
                                                u8 *lladdr, void *saddr,
                                                struct net_device *dev);

struct neigh_parms *neigh_parms_alloc(struct net_device *dev,
                                      struct neigh_table *tbl);
void neigh_parms_release(struct neigh_table *tbl, struct neigh_parms *parms);

static inline
struct net *neigh_parms_net(const struct neigh_parms *parms)
{
        return read_pnet(&parms->net);
}

unsigned long neigh_rand_reach_time(unsigned long base);

static inline void neigh_set_reach_time(struct neigh_parms *p)
{
        unsigned long base = NEIGH_VAR(p, BASE_REACHABLE_TIME);

        WRITE_ONCE(p->reachable_time, neigh_rand_reach_time(base));
}

void pneigh_enqueue(struct neigh_table *tbl, struct neigh_parms *p,
                    struct sk_buff *skb);
struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl, struct net *net,
                                   const void *key, struct net_device *dev);
int pneigh_create(struct neigh_table *tbl, struct net *net, const void *key,
                  struct net_device *dev, u32 flags, u8 protocol,
                  bool permanent);
int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *key,
                  struct net_device *dev);

static inline struct net *pneigh_net(const struct pneigh_entry *pneigh)
{
        return read_pnet(&pneigh->net);
}

void neigh_app_ns(struct neighbour *n);
void neigh_for_each(struct neigh_table *tbl,
                    void (*cb)(struct neighbour *, void *), void *cookie);
void __neigh_for_each_release(struct neigh_table *tbl,
                              int (*cb)(struct neighbour *));
int neigh_xmit(int fam, struct net_device *, const void *, struct sk_buff *);

struct neigh_seq_state {
        struct seq_net_private p;
        struct neigh_table *tbl;
        struct neigh_hash_table *nht;
        void *(*neigh_sub_iter)(struct neigh_seq_state *state,
                                struct neighbour *n, loff_t *pos);
        unsigned int bucket;
        unsigned int flags;
#define NEIGH_SEQ_NEIGH_ONLY        0x00000001
#define NEIGH_SEQ_IS_PNEIGH        0x00000002
#define NEIGH_SEQ_SKIP_NOARP        0x00000004
};
void *neigh_seq_start(struct seq_file *, loff_t *, struct neigh_table *,
                      unsigned int);
void *neigh_seq_next(struct seq_file *, void *, loff_t *);
void neigh_seq_stop(struct seq_file *, void *);

int neigh_proc_dointvec(const struct ctl_table *ctl, int write,
                        void *buffer, size_t *lenp, loff_t *ppos);
int neigh_proc_dointvec_jiffies(const struct ctl_table *ctl, int write,
                                void *buffer,
                                size_t *lenp, loff_t *ppos);
int neigh_proc_dointvec_ms_jiffies(const struct ctl_table *ctl, int write,
                                   void *buffer, size_t *lenp, loff_t *ppos);

int neigh_sysctl_register(struct net_device *dev, struct neigh_parms *p,
                          proc_handler *proc_handler);
void neigh_sysctl_unregister(struct neigh_parms *p);

static inline void __neigh_parms_put(struct neigh_parms *parms)
{
        refcount_dec(&parms->refcnt);
}

static inline struct neigh_parms *neigh_parms_clone(struct neigh_parms *parms)
{
        refcount_inc(&parms->refcnt);
        return parms;
}

/*
 *        Neighbour references
 */

static inline void neigh_release(struct neighbour *neigh)
{
        if (refcount_dec_and_test(&neigh->refcnt))
                neigh_destroy(neigh);
}

static inline struct neighbour * neigh_clone(struct neighbour *neigh)
{
        if (neigh)
                refcount_inc(&neigh->refcnt);
        return neigh;
}

#define neigh_hold(n)        refcount_inc(&(n)->refcnt)

static __always_inline int neigh_event_send_probe(struct neighbour *neigh,
                                                  struct sk_buff *skb,
                                                  const bool immediate_ok)
{
        unsigned long now = jiffies;

        if (READ_ONCE(neigh->used) != now)
                WRITE_ONCE(neigh->used, now);
        if (!(READ_ONCE(neigh->nud_state) & (NUD_CONNECTED | NUD_DELAY | NUD_PROBE)))
                return __neigh_event_send(neigh, skb, immediate_ok);
        return 0;
}

static inline int neigh_event_send(struct neighbour *neigh, struct sk_buff *skb)
{
        return neigh_event_send_probe(neigh, skb, true);
}

#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
static inline int neigh_hh_bridge(struct hh_cache *hh, struct sk_buff *skb)
{
        unsigned int seq, hh_alen = HH_DATA_ALIGN(ETH_HLEN);
        int err;

        err = skb_cow_head(skb, hh_alen);
        if (err)
                return err;

        do {
                seq = read_seqbegin(&hh->hh_lock);
                memcpy(skb->data - hh_alen, hh->hh_data, ETH_ALEN + hh_alen - ETH_HLEN);
        } while (read_seqretry(&hh->hh_lock, seq));
        return 0;
}
#endif

static inline int neigh_hh_output(const struct hh_cache *hh, struct sk_buff *skb)
{
        unsigned int hh_alen = 0;
        unsigned int seq;
        unsigned int hh_len;

        do {
                seq = read_seqbegin(&hh->hh_lock);
                hh_len = READ_ONCE(hh->hh_len);
                if (likely(hh_len <= HH_DATA_MOD)) {
                        hh_alen = HH_DATA_MOD;

                        /* skb_push() would proceed silently if we have room for
                         * the unaligned size but not for the aligned size:
                         * check headroom explicitly.
                         */
                        if (likely(skb_headroom(skb) >= HH_DATA_MOD)) {
                                /* this is inlined by gcc */
                                memcpy(skb->data - HH_DATA_MOD, hh->hh_data,
                                       HH_DATA_MOD);
                        }
                } else {
                        hh_alen = HH_DATA_ALIGN(hh_len);

                        if (likely(skb_headroom(skb) >= hh_alen)) {
                                memcpy(skb->data - hh_alen, hh->hh_data,
                                       hh_alen);
                        }
                }
        } while (read_seqretry(&hh->hh_lock, seq));

        if (WARN_ON_ONCE(skb_headroom(skb) < hh_alen)) {
                kfree_skb(skb);
                return NET_XMIT_DROP;
        }

        __skb_push(skb, hh_len);
        return dev_queue_xmit(skb);
}

static inline int neigh_output(struct neighbour *n, struct sk_buff *skb,
                               bool skip_cache)
{
        const struct hh_cache *hh = &n->hh;

        /* n->nud_state and hh->hh_len could be changed under us.
         * neigh_hh_output() is taking care of the race later.
         */
        if (!skip_cache &&
            (READ_ONCE(n->nud_state) & NUD_CONNECTED) &&
            READ_ONCE(hh->hh_len))
                return neigh_hh_output(hh, skb);

        return READ_ONCE(n->output)(n, skb);
}

static inline struct neighbour *
__neigh_lookup(struct neigh_table *tbl, const void *pkey, struct net_device *dev, int creat)
{
        struct neighbour *n = neigh_lookup(tbl, pkey, dev);

        if (n || !creat)
                return n;

        n = neigh_create(tbl, pkey, dev);
        return IS_ERR(n) ? NULL : n;
}

static inline struct neighbour *
__neigh_lookup_errno(struct neigh_table *tbl, const void *pkey,
  struct net_device *dev)
{
        struct neighbour *n = neigh_lookup(tbl, pkey, dev);

        if (n)
                return n;

        return neigh_create(tbl, pkey, dev);
}

struct neighbour_cb {
        unsigned long sched_next;
        unsigned int flags;
};

#define LOCALLY_ENQUEUED 0x1

#define NEIGH_CB(skb)        ((struct neighbour_cb *)(skb)->cb)

static inline void neigh_ha_snapshot(char *dst, const struct neighbour *n,
                                     const struct net_device *dev)
{
        unsigned int seq;

        do {
                seq = read_seqbegin(&n->ha_lock);
                memcpy(dst, n->ha, dev->addr_len);
        } while (read_seqretry(&n->ha_lock, seq));
}

static inline void neigh_update_is_router(struct neighbour *neigh, u32 flags,
                                          int *notify)
{
        u8 ndm_flags = 0;

        ndm_flags |= (flags & NEIGH_UPDATE_F_ISROUTER) ? NTF_ROUTER : 0;
        if ((neigh->flags ^ ndm_flags) & NTF_ROUTER) {
                if (ndm_flags & NTF_ROUTER)
                        neigh->flags |= NTF_ROUTER;
                else
                        neigh->flags &= ~NTF_ROUTER;
                *notify = 1;
        }
}
#endif














































































   12 





   12 



   12 


































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
/*
 * Copyright (c) 2006, 2020 Oracle and/or its affiliates.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/skbuff.h>
#include <linux/list.h>
#include <linux/errqueue.h>

#include "rds.h"

static unsigned int        rds_exthdr_size[__RDS_EXTHDR_MAX] = {
[RDS_EXTHDR_NONE]        = 0,
[RDS_EXTHDR_VERSION]        = sizeof(struct rds_ext_header_version),
[RDS_EXTHDR_RDMA]        = sizeof(struct rds_ext_header_rdma),
[RDS_EXTHDR_RDMA_DEST]        = sizeof(struct rds_ext_header_rdma_dest),
[RDS_EXTHDR_RDMA_BYTES] = sizeof(struct rds_ext_header_rdma_bytes),
[RDS_EXTHDR_NPATHS]        = sizeof(__be16),
[RDS_EXTHDR_GEN_NUM]        = sizeof(__be32),
[RDS_EXTHDR_SPORT_IDX]        = 1,
};

void rds_message_addref(struct rds_message *rm)
{
        rdsdebug("addref rm %p ref %d\n", rm, refcount_read(&rm->m_refcount));
        refcount_inc(&rm->m_refcount);
}
EXPORT_SYMBOL_GPL(rds_message_addref);

static inline bool rds_zcookie_add(struct rds_msg_zcopy_info *info, u32 cookie)
{
        struct rds_zcopy_cookies *ck = &info->zcookies;
        int ncookies = ck->num;

        if (ncookies == RDS_MAX_ZCOOKIES)
                return false;
        ck->cookies[ncookies] = cookie;
        ck->num =  ++ncookies;
        return true;
}

static struct rds_msg_zcopy_info *rds_info_from_znotifier(struct rds_znotifier *znotif)
{
        return container_of(znotif, struct rds_msg_zcopy_info, znotif);
}

void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *q)
{
        unsigned long flags;
        LIST_HEAD(copy);
        struct rds_msg_zcopy_info *info, *tmp;

        spin_lock_irqsave(&q->lock, flags);
        list_splice(&q->zcookie_head, &copy);
        INIT_LIST_HEAD(&q->zcookie_head);
        spin_unlock_irqrestore(&q->lock, flags);

        list_for_each_entry_safe(info, tmp, &copy, rs_zcookie_next) {
                list_del(&info->rs_zcookie_next);
                kfree(info);
        }
}

static void rds_rm_zerocopy_callback(struct rds_sock *rs,
                                     struct rds_znotifier *znotif)
{
        struct rds_msg_zcopy_info *info;
        struct rds_msg_zcopy_queue *q;
        u32 cookie = znotif->z_cookie;
        struct rds_zcopy_cookies *ck;
        struct list_head *head;
        unsigned long flags;

        mm_unaccount_pinned_pages(&znotif->z_mmp);
        q = &rs->rs_zcookie_queue;
        spin_lock_irqsave(&q->lock, flags);
        head = &q->zcookie_head;
        if (!list_empty(head)) {
                info = list_first_entry(head, struct rds_msg_zcopy_info,
                                        rs_zcookie_next);
                if (rds_zcookie_add(info, cookie)) {
                        spin_unlock_irqrestore(&q->lock, flags);
                        kfree(rds_info_from_znotifier(znotif));
                        /* caller invokes rds_wake_sk_sleep() */
                        return;
                }
        }

        info = rds_info_from_znotifier(znotif);
        ck = &info->zcookies;
        memset(ck, 0, sizeof(*ck));
        WARN_ON(!rds_zcookie_add(info, cookie));
        list_add_tail(&info->rs_zcookie_next, &q->zcookie_head);

        spin_unlock_irqrestore(&q->lock, flags);
        /* caller invokes rds_wake_sk_sleep() */
}

/*
 * This relies on dma_map_sg() not touching sg[].page during merging.
 */
static void rds_message_purge(struct rds_message *rm)
{
        struct rds_znotifier *znotifier;
        unsigned long i, flags;
        bool zcopy;

        if (unlikely(test_bit(RDS_MSG_PAGEVEC, &rm->m_flags)))
                return;

        spin_lock_irqsave(&rm->m_rs_lock, flags);
        znotifier = rm->data.op_mmp_znotifier;
        rm->data.op_mmp_znotifier = NULL;
        zcopy = !!znotifier;

        if (rm->m_rs) {
                struct rds_sock *rs = rm->m_rs;

                if (znotifier) {
                        rds_rm_zerocopy_callback(rs, znotifier);
                        rds_wake_sk_sleep(rs);
                }
                sock_put(rds_rs_to_sk(rs));
                rm->m_rs = NULL;
        } else if (znotifier) {
                /*
                 * Zerocopy can fail before the message is queued on the
                 * socket, so there is no rs to carry the notification.
                 */
                mm_unaccount_pinned_pages(&znotifier->z_mmp);
                kfree(rds_info_from_znotifier(znotifier));
        }
        spin_unlock_irqrestore(&rm->m_rs_lock, flags);

        for (i = 0; i < rm->data.op_nents; i++) {
                /* XXX will have to put_page for page refs */
                if (!zcopy)
                        __free_page(sg_page(&rm->data.op_sg[i]));
                else
                        put_page(sg_page(&rm->data.op_sg[i]));
        }
        rm->data.op_nents = 0;

        if (rm->rdma.op_active)
                rds_rdma_free_op(&rm->rdma);
        if (rm->rdma.op_rdma_mr)
                kref_put(&rm->rdma.op_rdma_mr->r_kref, __rds_put_mr_final);

        if (rm->atomic.op_active)
                rds_atomic_free_op(&rm->atomic);
        if (rm->atomic.op_rdma_mr)
                kref_put(&rm->atomic.op_rdma_mr->r_kref, __rds_put_mr_final);
}

void rds_message_put(struct rds_message *rm)
{
        rdsdebug("put rm %p ref %d\n", rm, refcount_read(&rm->m_refcount));
        WARN(!refcount_read(&rm->m_refcount), "danger refcount zero on %p\n", rm);
        if (refcount_dec_and_test(&rm->m_refcount)) {
                BUG_ON(!list_empty(&rm->m_sock_item));
                BUG_ON(!list_empty(&rm->m_conn_item));
                rds_message_purge(rm);

                kfree(rm);
        }
}
EXPORT_SYMBOL_GPL(rds_message_put);

void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
                                 __be16 dport, u64 seq)
{
        hdr->h_flags = 0;
        hdr->h_sport = sport;
        hdr->h_dport = dport;
        hdr->h_sequence = cpu_to_be64(seq);
        /* see rds_find_next_ext_space for reason why we memset the
         * ext header
         */
        memset(hdr->h_exthdr, RDS_EXTHDR_NONE, RDS_HEADER_EXT_SPACE);
}
EXPORT_SYMBOL_GPL(rds_message_populate_header);

/*
 * Find the next place we can add an RDS header extension with
 * specific length. Extension headers are pushed one after the
 * other. In the following, the number after the colon is the number
 * of bytes:
 *
 * [ type1:1 dta1:len1 [ type2:1 dta2:len2 ] ... ] RDS_EXTHDR_NONE
 *
 * If the extension headers fill the complete extension header space
 * (16 bytes), the trailing RDS_EXTHDR_NONE is omitted.
 */
static int rds_find_next_ext_space(struct rds_header *hdr, unsigned int len,
                                   u8 **ext_start)
{
        unsigned int ext_len;
        unsigned int type;
        int ind = 0;

        while ((ind + 1 + len) <= RDS_HEADER_EXT_SPACE) {
                if (hdr->h_exthdr[ind] == RDS_EXTHDR_NONE) {
                        *ext_start = hdr->h_exthdr + ind;
                        return 0;
                }

                type = hdr->h_exthdr[ind];

                ext_len = (type < __RDS_EXTHDR_MAX) ? rds_exthdr_size[type] : 0;
                WARN_ONCE(!ext_len, "Unknown ext hdr type %d\n", type);
                if (!ext_len)
                        return -EINVAL;

                /* ind points to a valid ext hdr with known length */
                ind += 1 + ext_len;
        }

        /* no room for extension */
        return -ENOSPC;
}

/* The ext hdr space is prefilled with zero from the kzalloc() */
int rds_message_add_extension(struct rds_header *hdr,
                              unsigned int type, const void *data)
{
        unsigned char *dst;
        unsigned int len;

        len = (type < __RDS_EXTHDR_MAX) ? rds_exthdr_size[type] : 0;
        if (!len)
                return 0;

        if (rds_find_next_ext_space(hdr, len, &dst))
                return 0;

        *dst++ = type;
        memcpy(dst, data, len);

        return 1;
}
EXPORT_SYMBOL_GPL(rds_message_add_extension);

/*
 * If a message has extension headers, retrieve them here.
 * Call like this:
 *
 * unsigned int pos = 0;
 *
 * while (1) {
 *        buflen = sizeof(buffer);
 *        type = rds_message_next_extension(hdr, &pos, buffer, &buflen);
 *        if (type == RDS_EXTHDR_NONE)
 *                break;
 *        ...
 * }
 */
int rds_message_next_extension(struct rds_header *hdr,
                unsigned int *pos, void *buf, unsigned int *buflen)
{
        unsigned int offset, ext_type, ext_len;
        u8 *src = hdr->h_exthdr;

        offset = *pos;
        if (offset >= RDS_HEADER_EXT_SPACE)
                goto none;

        /* Get the extension type and length. For now, the
         * length is implied by the extension type. */
        ext_type = src[offset++];

        if (ext_type == RDS_EXTHDR_NONE || ext_type >= __RDS_EXTHDR_MAX)
                goto none;
        ext_len = rds_exthdr_size[ext_type];
        if (offset + ext_len > RDS_HEADER_EXT_SPACE)
                goto none;

        *pos = offset + ext_len;
        if (ext_len < *buflen)
                *buflen = ext_len;
        memcpy(buf, src + offset, *buflen);
        return ext_type;

none:
        *pos = RDS_HEADER_EXT_SPACE;
        *buflen = 0;
        return RDS_EXTHDR_NONE;
}

int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset)
{
        struct rds_ext_header_rdma_dest ext_hdr;

        ext_hdr.h_rdma_rkey = cpu_to_be32(r_key);
        ext_hdr.h_rdma_offset = cpu_to_be32(offset);
        return rds_message_add_extension(hdr, RDS_EXTHDR_RDMA_DEST, &ext_hdr);
}
EXPORT_SYMBOL_GPL(rds_message_add_rdma_dest_extension);

/*
 * Each rds_message is allocated with extra space for the scatterlist entries
 * rds ops will need. This is to minimize memory allocation count. Then, each rds op
 * can grab SGs when initializing its part of the rds_message.
 */
struct rds_message *rds_message_alloc(unsigned int extra_len, gfp_t gfp)
{
        struct rds_message *rm;

        if (extra_len > KMALLOC_MAX_SIZE - sizeof(struct rds_message))
                return NULL;

        rm = kzalloc(sizeof(struct rds_message) + extra_len, gfp);
        if (!rm)
                goto out;

        rm->m_used_sgs = 0;
        rm->m_total_sgs = extra_len / sizeof(struct scatterlist);

        refcount_set(&rm->m_refcount, 1);
        INIT_LIST_HEAD(&rm->m_sock_item);
        INIT_LIST_HEAD(&rm->m_conn_item);
        spin_lock_init(&rm->m_rs_lock);
        init_waitqueue_head(&rm->m_flush_wait);

out:
        return rm;
}

/*
 * RDS ops use this to grab SG entries from the rm's sg pool.
 */
struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents)
{
        struct scatterlist *sg_first = (struct scatterlist *) &rm[1];
        struct scatterlist *sg_ret;

        if (nents <= 0) {
                pr_warn("rds: alloc sgs failed! nents <= 0\n");
                return ERR_PTR(-EINVAL);
        }

        if (rm->m_used_sgs + nents > rm->m_total_sgs) {
                pr_warn("rds: alloc sgs failed! total %d used %d nents %d\n",
                        rm->m_total_sgs, rm->m_used_sgs, nents);
                return ERR_PTR(-ENOMEM);
        }

        sg_ret = &sg_first[rm->m_used_sgs];
        sg_init_table(sg_ret, nents);
        rm->m_used_sgs += nents;

        return sg_ret;
}

struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len)
{
        struct rds_message *rm;
        unsigned int i;
        int num_sgs = DIV_ROUND_UP(total_len, PAGE_SIZE);
        int extra_bytes = num_sgs * sizeof(struct scatterlist);

        rm = rds_message_alloc(extra_bytes, GFP_NOWAIT);
        if (!rm)
                return ERR_PTR(-ENOMEM);

        set_bit(RDS_MSG_PAGEVEC, &rm->m_flags);
        rm->m_inc.i_hdr.h_len = cpu_to_be32(total_len);
        rm->data.op_nents = DIV_ROUND_UP(total_len, PAGE_SIZE);
        rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs);
        if (IS_ERR(rm->data.op_sg)) {
                void *err = ERR_CAST(rm->data.op_sg);
                rds_message_put(rm);
                return err;
        }

        for (i = 0; i < rm->data.op_nents; ++i) {
                sg_set_page(&rm->data.op_sg[i],
                                virt_to_page((void *)page_addrs[i]),
                                PAGE_SIZE, 0);
        }

        return rm;
}

static int rds_message_zcopy_from_user(struct rds_message *rm, struct iov_iter *from)
{
        struct scatterlist *sg;
        int ret = 0;
        int length = iov_iter_count(from);
        struct rds_msg_zcopy_info *info;

        rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));

        /*
         * now allocate and copy in the data payload.
         */
        sg = rm->data.op_sg;

        info = kzalloc_obj(*info);
        if (!info)
                return -ENOMEM;
        INIT_LIST_HEAD(&info->rs_zcookie_next);
        rm->data.op_mmp_znotifier = &info->znotif;
        if (mm_account_pinned_pages(&rm->data.op_mmp_znotifier->z_mmp,
                                    length)) {
                ret = -ENOMEM;
                goto err;
        }
        while (iov_iter_count(from)) {
                struct page *pages;
                size_t start;
                ssize_t copied;

                copied = iov_iter_get_pages2(from, &pages, PAGE_SIZE,
                                            1, &start);
                if (copied < 0) {
                        struct mmpin *mmp;
                        int i;

                        for (i = 0; i < rm->data.op_nents; i++)
                                put_page(sg_page(&rm->data.op_sg[i]));
                        rm->data.op_nents = 0;
                        mmp = &rm->data.op_mmp_znotifier->z_mmp;
                        mm_unaccount_pinned_pages(mmp);
                        ret = -EFAULT;
                        goto err;
                }
                length -= copied;
                sg_set_page(sg, pages, copied, start);
                rm->data.op_nents++;
                sg++;
        }
        WARN_ON_ONCE(length != 0);
        return ret;
err:
        kfree(info);
        rm->data.op_mmp_znotifier = NULL;
        return ret;
}

int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
                               bool zcopy)
{
        unsigned long to_copy, nbytes;
        unsigned long sg_off;
        struct scatterlist *sg;
        int ret = 0;

        rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));

        /* now allocate and copy in the data payload.  */
        sg = rm->data.op_sg;
        sg_off = 0; /* Dear gcc, sg->page will be null from kzalloc. */

        if (zcopy)
                return rds_message_zcopy_from_user(rm, from);

        while (iov_iter_count(from)) {
                if (!sg_page(sg)) {
                        ret = rds_page_remainder_alloc(sg, iov_iter_count(from),
                                                       GFP_HIGHUSER);
                        if (ret)
                                return ret;
                        rm->data.op_nents++;
                        sg_off = 0;
                }

                to_copy = min_t(unsigned long, iov_iter_count(from),
                                sg->length - sg_off);

                rds_stats_add(s_copy_from_user, to_copy);
                nbytes = copy_page_from_iter(sg_page(sg), sg->offset + sg_off,
                                             to_copy, from);
                if (nbytes != to_copy)
                        return -EFAULT;

                sg_off += to_copy;

                if (sg_off == sg->length)
                        sg++;
        }

        return ret;
}

int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to)
{
        struct rds_message *rm;
        struct scatterlist *sg;
        unsigned long to_copy;
        unsigned long vec_off;
        int copied;
        int ret;
        u32 len;

        rm = container_of(inc, struct rds_message, m_inc);
        len = be32_to_cpu(rm->m_inc.i_hdr.h_len);

        sg = rm->data.op_sg;
        vec_off = 0;
        copied = 0;

        while (iov_iter_count(to) && copied < len) {
                to_copy = min_t(unsigned long, iov_iter_count(to),
                                sg->length - vec_off);
                to_copy = min_t(unsigned long, to_copy, len - copied);

                rds_stats_add(s_copy_to_user, to_copy);
                ret = copy_page_to_iter(sg_page(sg), sg->offset + vec_off,
                                        to_copy, to);
                if (ret != to_copy)
                        return -EFAULT;

                vec_off += to_copy;
                copied += to_copy;

                if (vec_off == sg->length) {
                        vec_off = 0;
                        sg++;
                }
        }

        return copied;
}

/*
 * If the message is still on the send queue, wait until the transport
 * is done with it. This is particularly important for RDMA operations.
 */
void rds_message_wait(struct rds_message *rm)
{
        wait_event_interruptible(rm->m_flush_wait,
                        !test_bit(RDS_MSG_MAPPED, &rm->m_flags));
}

void rds_message_unmapped(struct rds_message *rm)
{
        clear_bit(RDS_MSG_MAPPED, &rm->m_flags);
        wake_up_interruptible(&rm->m_flush_wait);
}
EXPORT_SYMBOL_GPL(rds_message_unmapped);




























  544 















    2 
    2 






































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM pagemap

#if !defined(_TRACE_PAGEMAP_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_PAGEMAP_H

#include <linux/tracepoint.h>
#include <linux/mm.h>

#define        PAGEMAP_MAPPED                0x0001u
#define PAGEMAP_ANONYMOUS        0x0002u
#define PAGEMAP_FILE                0x0004u
#define PAGEMAP_SWAPCACHE        0x0008u
#define PAGEMAP_SWAPBACKED        0x0010u
#define PAGEMAP_MAPPEDDISK        0x0020u
#define PAGEMAP_BUFFERS                0x0040u

#define trace_pagemap_flags(folio) ( \
        (folio_test_anon(folio)                ? PAGEMAP_ANONYMOUS  : PAGEMAP_FILE) | \
        (folio_mapped(folio)                ? PAGEMAP_MAPPED     : 0) | \
        (folio_test_swapcache(folio)        ? PAGEMAP_SWAPCACHE  : 0) | \
        (folio_test_swapbacked(folio)        ? PAGEMAP_SWAPBACKED : 0) | \
        (folio_test_mappedtodisk(folio)        ? PAGEMAP_MAPPEDDISK : 0) | \
        (folio_test_private(folio)        ? PAGEMAP_BUFFERS    : 0) \
        )

TRACE_EVENT(mm_lru_insertion,

        TP_PROTO(struct folio *folio),

        TP_ARGS(folio),

        TP_STRUCT__entry(
                __field(struct folio *,        folio        )
                __field(unsigned long,        pfn        )
                __field(enum lru_list,        lru        )
                __field(unsigned long,        flags        )
        ),

        TP_fast_assign(
                __entry->folio        = folio;
                __entry->pfn        = folio_pfn(folio);
                __entry->lru        = folio_lru_list(folio);
                __entry->flags        = trace_pagemap_flags(folio);
        ),

        /* Flag format is based on page-types.c formatting for pagemap */
        TP_printk("folio=%p pfn=0x%lx lru=%d flags=%s%s%s%s%s%s",
                        __entry->folio,
                        __entry->pfn,
                        __entry->lru,
                        __entry->flags & PAGEMAP_MAPPED                ? "M" : " ",
                        __entry->flags & PAGEMAP_ANONYMOUS        ? "a" : "f",
                        __entry->flags & PAGEMAP_SWAPCACHE        ? "s" : " ",
                        __entry->flags & PAGEMAP_SWAPBACKED        ? "b" : " ",
                        __entry->flags & PAGEMAP_MAPPEDDISK        ? "d" : " ",
                        __entry->flags & PAGEMAP_BUFFERS        ? "B" : " ")
);

TRACE_EVENT(mm_lru_activate,

        TP_PROTO(struct folio *folio),

        TP_ARGS(folio),

        TP_STRUCT__entry(
                __field(struct folio *,        folio        )
                __field(unsigned long,        pfn        )
        ),

        TP_fast_assign(
                __entry->folio        = folio;
                __entry->pfn        = folio_pfn(folio);
        ),

        TP_printk("folio=%p pfn=0x%lx", __entry->folio, __entry->pfn)
);

#endif /* _TRACE_PAGEMAP_H */

/* This part must be outside protection */
#include <trace/define_trace.h>































































































































































































































































































































































































































































































































































































































































































































































































  140 














  140 




















































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
// SPDX-License-Identifier: GPL-2.0
/*
 * security/tomoyo/condition.c
 *
 * Copyright (C) 2005-2011  NTT DATA CORPORATION
 */

#include "common.h"
#include <linux/slab.h>

/* List of "struct tomoyo_condition". */
LIST_HEAD(tomoyo_condition_list);

/**
 * tomoyo_argv - Check argv[] in "struct linux_binbrm".
 *
 * @index:   Index number of @arg_ptr.
 * @arg_ptr: Contents of argv[@index].
 * @argc:    Length of @argv.
 * @argv:    Pointer to "struct tomoyo_argv".
 * @checked: Set to true if @argv[@index] was found.
 *
 * Returns true on success, false otherwise.
 */
static bool tomoyo_argv(const unsigned int index, const char *arg_ptr,
                        const int argc, const struct tomoyo_argv *argv,
                        u8 *checked)
{
        int i;
        struct tomoyo_path_info arg;

        arg.name = arg_ptr;
        for (i = 0; i < argc; argv++, checked++, i++) {
                bool result;

                if (index != argv->index)
                        continue;
                *checked = 1;
                tomoyo_fill_path_info(&arg);
                result = tomoyo_path_matches_pattern(&arg, argv->value);
                if (argv->is_not)
                        result = !result;
                if (!result)
                        return false;
        }
        return true;
}

/**
 * tomoyo_envp - Check envp[] in "struct linux_binbrm".
 *
 * @env_name:  The name of environment variable.
 * @env_value: The value of environment variable.
 * @envc:      Length of @envp.
 * @envp:      Pointer to "struct tomoyo_envp".
 * @checked:   Set to true if @envp[@env_name] was found.
 *
 * Returns true on success, false otherwise.
 */
static bool tomoyo_envp(const char *env_name, const char *env_value,
                        const int envc, const struct tomoyo_envp *envp,
                        u8 *checked)
{
        int i;
        struct tomoyo_path_info name;
        struct tomoyo_path_info value;

        name.name = env_name;
        tomoyo_fill_path_info(&name);
        value.name = env_value;
        tomoyo_fill_path_info(&value);
        for (i = 0; i < envc; envp++, checked++, i++) {
                bool result;

                if (!tomoyo_path_matches_pattern(&name, envp->name))
                        continue;
                *checked = 1;
                if (envp->value) {
                        result = tomoyo_path_matches_pattern(&value,
                                                             envp->value);
                        if (envp->is_not)
                                result = !result;
                } else {
                        result = true;
                        if (!envp->is_not)
                                result = !result;
                }
                if (!result)
                        return false;
        }
        return true;
}

/**
 * tomoyo_scan_bprm - Scan "struct linux_binprm".
 *
 * @ee:   Pointer to "struct tomoyo_execve".
 * @argc: Length of @argc.
 * @argv: Pointer to "struct tomoyo_argv".
 * @envc: Length of @envp.
 * @envp: Pointer to "struct tomoyo_envp".
 *
 * Returns true on success, false otherwise.
 */
static bool tomoyo_scan_bprm(struct tomoyo_execve *ee,
                             const u16 argc, const struct tomoyo_argv *argv,
                             const u16 envc, const struct tomoyo_envp *envp)
{
        struct linux_binprm *bprm = ee->bprm;
        struct tomoyo_page_dump *dump = &ee->dump;
        char *arg_ptr = ee->tmp;
        int arg_len = 0;
        unsigned long pos = bprm->p;
        int offset = pos % PAGE_SIZE;
        int argv_count = bprm->argc;
        int envp_count = bprm->envc;
        bool result = true;
        u8 local_checked[32];
        u8 *checked;

        if (argc + envc <= sizeof(local_checked)) {
                checked = local_checked;
                memset(local_checked, 0, sizeof(local_checked));
        } else {
                checked = kzalloc(argc + envc, GFP_NOFS);
                if (!checked)
                        return false;
        }
        while (argv_count || envp_count) {
                if (!tomoyo_dump_page(bprm, pos, dump)) {
                        result = false;
                        goto out;
                }
                pos += PAGE_SIZE - offset;
                while (offset < PAGE_SIZE) {
                        /* Read. */
                        const char *kaddr = dump->data;
                        const unsigned char c = kaddr[offset++];

                        if (c && arg_len < TOMOYO_EXEC_TMPSIZE - 10) {
                                if (c == '\\') {
                                        arg_ptr[arg_len++] = '\\';
                                        arg_ptr[arg_len++] = '\\';
                                } else if (c > ' ' && c < 127) {
                                        arg_ptr[arg_len++] = c;
                                } else {
                                        arg_ptr[arg_len++] = '\\';
                                        arg_ptr[arg_len++] = (c >> 6) + '0';
                                        arg_ptr[arg_len++] =
                                                ((c >> 3) & 7) + '0';
                                        arg_ptr[arg_len++] = (c & 7) + '0';
                                }
                        } else {
                                arg_ptr[arg_len] = '\0';
                        }
                        if (c)
                                continue;
                        /* Check. */
                        if (argv_count) {
                                if (!tomoyo_argv(bprm->argc - argv_count,
                                                 arg_ptr, argc, argv,
                                                 checked)) {
                                        result = false;
                                        break;
                                }
                                argv_count--;
                        } else if (envp_count) {
                                char *cp = strchr(arg_ptr, '=');

                                if (cp) {
                                        *cp = '\0';
                                        if (!tomoyo_envp(arg_ptr, cp + 1,
                                                         envc, envp,
                                                         checked + argc)) {
                                                result = false;
                                                break;
                                        }
                                }
                                envp_count--;
                        } else {
                                break;
                        }
                        arg_len = 0;
                }
                offset = 0;
                if (!result)
                        break;
        }
out:
        if (result) {
                int i;

                /* Check not-yet-checked entries. */
                for (i = 0; i < argc; i++) {
                        if (checked[i])
                                continue;
                        /*
                         * Return true only if all unchecked indexes in
                         * bprm->argv[] are not matched.
                         */
                        if (argv[i].is_not)
                                continue;
                        result = false;
                        break;
                }
                for (i = 0; i < envc; envp++, i++) {
                        if (checked[argc + i])
                                continue;
                        /*
                         * Return true only if all unchecked environ variables
                         * in bprm->envp[] are either undefined or not matched.
                         */
                        if ((!envp->value && !envp->is_not) ||
                            (envp->value && envp->is_not))
                                continue;
                        result = false;
                        break;
                }
        }
        if (checked != local_checked)
                kfree(checked);
        return result;
}

/**
 * tomoyo_scan_exec_realpath - Check "exec.realpath" parameter of "struct tomoyo_condition".
 *
 * @file:  Pointer to "struct file".
 * @ptr:   Pointer to "struct tomoyo_name_union".
 * @match: True if "exec.realpath=", false if "exec.realpath!=".
 *
 * Returns true on success, false otherwise.
 */
static bool tomoyo_scan_exec_realpath(struct file *file,
                                      const struct tomoyo_name_union *ptr,
                                      const bool match)
{
        bool result;
        struct tomoyo_path_info exe;

        if (!file)
                return false;
        exe.name = tomoyo_realpath_from_path(&file->f_path);
        if (!exe.name)
                return false;
        tomoyo_fill_path_info(&exe);
        result = tomoyo_compare_name_union(&exe, ptr);
        kfree(exe.name);
        return result == match;
}

/**
 * tomoyo_get_dqword - tomoyo_get_name() for a quoted string.
 *
 * @start: String to save.
 *
 * Returns pointer to "struct tomoyo_path_info" on success, NULL otherwise.
 */
static const struct tomoyo_path_info *tomoyo_get_dqword(char *start)
{
        char *cp = start + strlen(start) - 1;

        if (cp == start || *start++ != '"' || *cp != '"')
                return NULL;
        *cp = '\0';
        if (*start && !tomoyo_correct_word(start))
                return NULL;
        return tomoyo_get_name(start);
}

/**
 * tomoyo_parse_name_union_quoted - Parse a quoted word.
 *
 * @param: Pointer to "struct tomoyo_acl_param".
 * @ptr:   Pointer to "struct tomoyo_name_union".
 *
 * Returns true on success, false otherwise.
 */
static bool tomoyo_parse_name_union_quoted(struct tomoyo_acl_param *param,
                                           struct tomoyo_name_union *ptr)
{
        char *filename = param->data;

        if (*filename == '@')
                return tomoyo_parse_name_union(param, ptr);
        ptr->filename = tomoyo_get_dqword(filename);
        return ptr->filename != NULL;
}

/**
 * tomoyo_parse_argv - Parse an argv[] condition part.
 *
 * @left:  Lefthand value.
 * @right: Righthand value.
 * @argv:  Pointer to "struct tomoyo_argv".
 *
 * Returns true on success, false otherwise.
 */
static bool tomoyo_parse_argv(char *left, char *right,
                              struct tomoyo_argv *argv)
{
        if (tomoyo_parse_ulong(&argv->index, &left) !=
            TOMOYO_VALUE_TYPE_DECIMAL || *left++ != ']' || *left)
                return false;
        argv->value = tomoyo_get_dqword(right);
        return argv->value != NULL;
}

/**
 * tomoyo_parse_envp - Parse an envp[] condition part.
 *
 * @left:  Lefthand value.
 * @right: Righthand value.
 * @envp:  Pointer to "struct tomoyo_envp".
 *
 * Returns true on success, false otherwise.
 */
static bool tomoyo_parse_envp(char *left, char *right,
                              struct tomoyo_envp *envp)
{
        const struct tomoyo_path_info *name;
        const struct tomoyo_path_info *value;
        char *cp = left + strlen(left) - 1;

        if (*cp-- != ']' || *cp != '"')
                goto out;
        *cp = '\0';
        if (!tomoyo_correct_word(left))
                goto out;
        name = tomoyo_get_name(left);
        if (!name)
                goto out;
        if (!strcmp(right, "NULL")) {
                value = NULL;
        } else {
                value = tomoyo_get_dqword(right);
                if (!value) {
                        tomoyo_put_name(name);
                        goto out;
                }
        }
        envp->name = name;
        envp->value = value;
        return true;
out:
        return false;
}

/**
 * tomoyo_same_condition - Check for duplicated "struct tomoyo_condition" entry.
 *
 * @a: Pointer to "struct tomoyo_condition".
 * @b: Pointer to "struct tomoyo_condition".
 *
 * Returns true if @a == @b, false otherwise.
 */
static inline bool tomoyo_same_condition(const struct tomoyo_condition *a,
                                         const struct tomoyo_condition *b)
{
        return a->size == b->size && a->condc == b->condc &&
                a->numbers_count == b->numbers_count &&
                a->names_count == b->names_count &&
                a->argc == b->argc && a->envc == b->envc &&
                a->grant_log == b->grant_log && a->transit == b->transit &&
                !memcmp(a + 1, b + 1, a->size - sizeof(*a));
}

/**
 * tomoyo_condition_type - Get condition type.
 *
 * @word: Keyword string.
 *
 * Returns one of values in "enum tomoyo_conditions_index" on success,
 * TOMOYO_MAX_CONDITION_KEYWORD otherwise.
 */
static u8 tomoyo_condition_type(const char *word)
{
        u8 i;

        for (i = 0; i < TOMOYO_MAX_CONDITION_KEYWORD; i++) {
                if (!strcmp(word, tomoyo_condition_keyword[i]))
                        break;
        }
        return i;
}

/* Define this to enable debug mode. */
/* #define DEBUG_CONDITION */

#ifdef DEBUG_CONDITION
#define dprintk printk
#else
#define dprintk(...) do { } while (0)
#endif

/**
 * tomoyo_commit_condition - Commit "struct tomoyo_condition".
 *
 * @entry: Pointer to "struct tomoyo_condition".
 *
 * Returns pointer to "struct tomoyo_condition" on success, NULL otherwise.
 *
 * This function merges duplicated entries. This function returns NULL if
 * @entry is not duplicated but memory quota for policy has exceeded.
 */
static struct tomoyo_condition *tomoyo_commit_condition
(struct tomoyo_condition *entry)
{
        struct tomoyo_condition *ptr;
        bool found = false;

        if (mutex_lock_interruptible(&tomoyo_policy_lock)) {
                dprintk(KERN_WARNING "%u: %s failed\n", __LINE__, __func__);
                ptr = NULL;
                found = true;
                goto out;
        }
        list_for_each_entry(ptr, &tomoyo_condition_list, head.list) {
                if (!tomoyo_same_condition(ptr, entry) ||
                    atomic_read(&ptr->head.users) == TOMOYO_GC_IN_PROGRESS)
                        continue;
                /* Same entry found. Share this entry. */
                atomic_inc(&ptr->head.users);
                found = true;
                break;
        }
        if (!found) {
                if (tomoyo_memory_ok(entry)) {
                        atomic_set(&entry->head.users, 1);
                        list_add(&entry->head.list, &tomoyo_condition_list);
                } else {
                        found = true;
                        ptr = NULL;
                }
        }
        mutex_unlock(&tomoyo_policy_lock);
out:
        if (found) {
                tomoyo_del_condition(&entry->head.list);
                kfree(entry);
                entry = ptr;
        }
        return entry;
}

/**
 * tomoyo_get_transit_preference - Parse domain transition preference for execve().
 *
 * @param: Pointer to "struct tomoyo_acl_param".
 * @e:     Pointer to "struct tomoyo_condition".
 *
 * Returns the condition string part.
 */
static char *tomoyo_get_transit_preference(struct tomoyo_acl_param *param,
                                           struct tomoyo_condition *e)
{
        char * const pos = param->data;
        bool flag;

        if (*pos == '<') {
                e->transit = tomoyo_get_domainname(param);
                goto done;
        }
        {
                char *cp = strchr(pos, ' ');

                if (cp)
                        *cp = '\0';
                flag = tomoyo_correct_path(pos) || !strcmp(pos, "keep") ||
                        !strcmp(pos, "initialize") || !strcmp(pos, "reset") ||
                        !strcmp(pos, "child") || !strcmp(pos, "parent");
                if (cp)
                        *cp = ' ';
        }
        if (!flag)
                return pos;
        e->transit = tomoyo_get_name(tomoyo_read_token(param));
done:
        if (e->transit)
                return param->data;
        /*
         * Return a bad read-only condition string that will let
         * tomoyo_get_condition() return NULL.
         */
        return "/";
}

/**
 * tomoyo_get_condition - Parse condition part.
 *
 * @param: Pointer to "struct tomoyo_acl_param".
 *
 * Returns pointer to "struct tomoyo_condition" on success, NULL otherwise.
 */
struct tomoyo_condition *tomoyo_get_condition(struct tomoyo_acl_param *param)
{
        struct tomoyo_condition *entry = NULL;
        struct tomoyo_condition_element *condp = NULL;
        struct tomoyo_number_union *numbers_p = NULL;
        struct tomoyo_name_union *names_p = NULL;
        struct tomoyo_argv *argv = NULL;
        struct tomoyo_envp *envp = NULL;
        struct tomoyo_condition e = { };
        char * const start_of_string =
                tomoyo_get_transit_preference(param, &e);
        char * const end_of_string = start_of_string + strlen(start_of_string);
        char *pos;

rerun:
        pos = start_of_string;
        while (1) {
                u8 left = -1;
                u8 right = -1;
                char *left_word = pos;
                char *cp;
                char *right_word;
                bool is_not;

                if (!*left_word)
                        break;
                /*
                 * Since left-hand condition does not allow use of "path_group"
                 * or "number_group" and environment variable's names do not
                 * accept '=', it is guaranteed that the original line consists
                 * of one or more repetition of $left$operator$right blocks
                 * where "$left is free from '=' and ' '" and "$operator is
                 * either '=' or '!='" and "$right is free from ' '".
                 * Therefore, we can reconstruct the original line at the end
                 * of dry run even if we overwrite $operator with '\0'.
                 */
                cp = strchr(pos, ' ');
                if (cp) {
                        *cp = '\0'; /* Will restore later. */
                        pos = cp + 1;
                } else {
                        pos = "";
                }
                right_word = strchr(left_word, '=');
                if (!right_word || right_word == left_word)
                        goto out;
                is_not = *(right_word - 1) == '!';
                if (is_not)
                        *(right_word++ - 1) = '\0'; /* Will restore later. */
                else if (*(right_word + 1) != '=')
                        *right_word++ = '\0'; /* Will restore later. */
                else
                        goto out;
                dprintk(KERN_WARNING "%u: <%s>%s=<%s>\n", __LINE__, left_word,
                        is_not ? "!" : "", right_word);
                if (!strcmp(left_word, "grant_log")) {
                        if (entry) {
                                if (is_not ||
                                    entry->grant_log != TOMOYO_GRANTLOG_AUTO)
                                        goto out;
                                else if (!strcmp(right_word, "yes"))
                                        entry->grant_log = TOMOYO_GRANTLOG_YES;
                                else if (!strcmp(right_word, "no"))
                                        entry->grant_log = TOMOYO_GRANTLOG_NO;
                                else
                                        goto out;
                        }
                        continue;
                }
                if (!strncmp(left_word, "exec.argv[", 10)) {
                        if (!argv) {
                                e.argc++;
                                e.condc++;
                        } else {
                                e.argc--;
                                e.condc--;
                                left = TOMOYO_ARGV_ENTRY;
                                argv->is_not = is_not;
                                if (!tomoyo_parse_argv(left_word + 10,
                                                       right_word, argv++))
                                        goto out;
                        }
                        goto store_value;
                }
                if (!strncmp(left_word, "exec.envp[\"", 11)) {
                        if (!envp) {
                                e.envc++;
                                e.condc++;
                        } else {
                                e.envc--;
                                e.condc--;
                                left = TOMOYO_ENVP_ENTRY;
                                envp->is_not = is_not;
                                if (!tomoyo_parse_envp(left_word + 11,
                                                       right_word, envp++))
                                        goto out;
                        }
                        goto store_value;
                }
                left = tomoyo_condition_type(left_word);
                dprintk(KERN_WARNING "%u: <%s> left=%u\n", __LINE__, left_word,
                        left);
                if (left == TOMOYO_MAX_CONDITION_KEYWORD) {
                        if (!numbers_p) {
                                e.numbers_count++;
                        } else {
                                e.numbers_count--;
                                left = TOMOYO_NUMBER_UNION;
                                param->data = left_word;
                                if (*left_word == '@' ||
                                    !tomoyo_parse_number_union(param,
                                                               numbers_p++))
                                        goto out;
                        }
                }
                if (!condp)
                        e.condc++;
                else
                        e.condc--;
                if (left == TOMOYO_EXEC_REALPATH ||
                    left == TOMOYO_SYMLINK_TARGET) {
                        if (!names_p) {
                                e.names_count++;
                        } else {
                                e.names_count--;
                                right = TOMOYO_NAME_UNION;
                                param->data = right_word;
                                if (!tomoyo_parse_name_union_quoted(param,
                                                                    names_p++))
                                        goto out;
                        }
                        goto store_value;
                }
                right = tomoyo_condition_type(right_word);
                if (right == TOMOYO_MAX_CONDITION_KEYWORD) {
                        if (!numbers_p) {
                                e.numbers_count++;
                        } else {
                                e.numbers_count--;
                                right = TOMOYO_NUMBER_UNION;
                                param->data = right_word;
                                if (!tomoyo_parse_number_union(param,
                                                               numbers_p++))
                                        goto out;
                        }
                }
store_value:
                if (!condp) {
                        dprintk(KERN_WARNING "%u: dry_run left=%u right=%u match=%u\n",
                                __LINE__, left, right, !is_not);
                        continue;
                }
                condp->left = left;
                condp->right = right;
                condp->equals = !is_not;
                dprintk(KERN_WARNING "%u: left=%u right=%u match=%u\n",
                        __LINE__, condp->left, condp->right,
                        condp->equals);
                condp++;
        }
        dprintk(KERN_INFO "%u: cond=%u numbers=%u names=%u ac=%u ec=%u\n",
                __LINE__, e.condc, e.numbers_count, e.names_count, e.argc,
                e.envc);
        if (entry) {
                BUG_ON(e.names_count | e.numbers_count | e.argc | e.envc |
                       e.condc);
                return tomoyo_commit_condition(entry);
        }
        e.size = sizeof(*entry)
                + e.condc * sizeof(struct tomoyo_condition_element)
                + e.numbers_count * sizeof(struct tomoyo_number_union)
                + e.names_count * sizeof(struct tomoyo_name_union)
                + e.argc * sizeof(struct tomoyo_argv)
                + e.envc * sizeof(struct tomoyo_envp);
        entry = kzalloc(e.size, GFP_NOFS);
        if (!entry)
                goto out2;
        *entry = e;
        e.transit = NULL;
        condp = (struct tomoyo_condition_element *) (entry + 1);
        numbers_p = (struct tomoyo_number_union *) (condp + e.condc);
        names_p = (struct tomoyo_name_union *) (numbers_p + e.numbers_count);
        argv = (struct tomoyo_argv *) (names_p + e.names_count);
        envp = (struct tomoyo_envp *) (argv + e.argc);
        {
                bool flag = false;

                for (pos = start_of_string; pos < end_of_string; pos++) {
                        if (*pos)
                                continue;
                        if (flag) /* Restore " ". */
                                *pos = ' ';
                        else if (*(pos + 1) == '=') /* Restore "!=". */
                                *pos = '!';
                        else /* Restore "=". */
                                *pos = '=';
                        flag = !flag;
                }
        }
        goto rerun;
out:
        dprintk(KERN_WARNING "%u: %s failed\n", __LINE__, __func__);
        if (entry) {
                tomoyo_del_condition(&entry->head.list);
                kfree(entry);
        }
out2:
        tomoyo_put_name(e.transit);
        return NULL;
}

/**
 * tomoyo_get_attributes - Revalidate "struct inode".
 *
 * @obj: Pointer to "struct tomoyo_obj_info".
 *
 * Returns nothing.
 */
void tomoyo_get_attributes(struct tomoyo_obj_info *obj)
{
        u8 i;
        struct dentry *dentry = NULL;

        for (i = 0; i < TOMOYO_MAX_PATH_STAT; i++) {
                struct inode *inode;

                switch (i) {
                case TOMOYO_PATH1:
                        dentry = obj->path1.dentry;
                        if (!dentry)
                                continue;
                        break;
                case TOMOYO_PATH2:
                        dentry = obj->path2.dentry;
                        if (!dentry)
                                continue;
                        break;
                default:
                        if (!dentry)
                                continue;
                        dentry = dget_parent(dentry);
                        break;
                }
                inode = d_backing_inode(dentry);
                if (inode) {
                        struct tomoyo_mini_stat *stat = &obj->stat[i];

                        stat->uid  = inode->i_uid;
                        stat->gid  = inode->i_gid;
                        stat->ino  = inode->i_ino;
                        stat->mode = inode->i_mode;
                        stat->dev  = inode->i_sb->s_dev;
                        stat->rdev = inode->i_rdev;
                        obj->stat_valid[i] = true;
                }
                if (i & 1) /* TOMOYO_PATH1_PARENT or TOMOYO_PATH2_PARENT */
                        dput(dentry);
        }
}

/**
 * tomoyo_condition - Check condition part.
 *
 * @r:    Pointer to "struct tomoyo_request_info".
 * @cond: Pointer to "struct tomoyo_condition". Maybe NULL.
 *
 * Returns true on success, false otherwise.
 *
 * Caller holds tomoyo_read_lock().
 */
bool tomoyo_condition(struct tomoyo_request_info *r,
                      const struct tomoyo_condition *cond)
{
        u32 i;
        u64 min_v[2] = { 0, 0 };
        u64 max_v[2] = { 0, 0 };
        const struct tomoyo_condition_element *condp;
        const struct tomoyo_number_union *numbers_p;
        const struct tomoyo_name_union *names_p;
        const struct tomoyo_argv *argv;
        const struct tomoyo_envp *envp;
        struct tomoyo_obj_info *obj;
        u16 condc;
        u16 argc;
        u16 envc;
        struct linux_binprm *bprm = NULL;

        if (!cond)
                return true;
        condc = cond->condc;
        argc = cond->argc;
        envc = cond->envc;
        obj = r->obj;
        if (r->ee)
                bprm = r->ee->bprm;
        if (!bprm && (argc || envc))
                return false;
        condp = (struct tomoyo_condition_element *) (cond + 1);
        numbers_p = (const struct tomoyo_number_union *) (condp + condc);
        names_p = (const struct tomoyo_name_union *)
                (numbers_p + cond->numbers_count);
        argv = (const struct tomoyo_argv *) (names_p + cond->names_count);
        envp = (const struct tomoyo_envp *) (argv + argc);
        for (i = 0; i < condc; i++) {
                const bool match = condp->equals;
                const u8 left = condp->left;
                const u8 right = condp->right;
                bool is_bitop[2] = { false, false };
                u8 j;

                condp++;
                /* Check argv[] and envp[] later. */
                if (left == TOMOYO_ARGV_ENTRY || left == TOMOYO_ENVP_ENTRY)
                        continue;
                /* Check string expressions. */
                if (right == TOMOYO_NAME_UNION) {
                        const struct tomoyo_name_union *ptr = names_p++;
                        struct tomoyo_path_info *symlink;
                        struct tomoyo_execve *ee;
                        struct file *file;

                        switch (left) {
                        case TOMOYO_SYMLINK_TARGET:
                                symlink = obj ? obj->symlink_target : NULL;
                                if (!symlink ||
                                    !tomoyo_compare_name_union(symlink, ptr)
                                    == match)
                                        goto out;
                                break;
                        case TOMOYO_EXEC_REALPATH:
                                ee = r->ee;
                                file = ee ? ee->bprm->file : NULL;
                                if (!tomoyo_scan_exec_realpath(file, ptr,
                                                               match))
                                        goto out;
                                break;
                        }
                        continue;
                }
                /* Check numeric or bit-op expressions. */
                for (j = 0; j < 2; j++) {
                        const u8 index = j ? right : left;
                        u64 value = 0;

                        switch (index) {
                        case TOMOYO_TASK_UID:
                                value = from_kuid(&init_user_ns, current_uid());
                                break;
                        case TOMOYO_TASK_EUID:
                                value = from_kuid(&init_user_ns, current_euid());
                                break;
                        case TOMOYO_TASK_SUID:
                                value = from_kuid(&init_user_ns, current_suid());
                                break;
                        case TOMOYO_TASK_FSUID:
                                value = from_kuid(&init_user_ns, current_fsuid());
                                break;
                        case TOMOYO_TASK_GID:
                                value = from_kgid(&init_user_ns, current_gid());
                                break;
                        case TOMOYO_TASK_EGID:
                                value = from_kgid(&init_user_ns, current_egid());
                                break;
                        case TOMOYO_TASK_SGID:
                                value = from_kgid(&init_user_ns, current_sgid());
                                break;
                        case TOMOYO_TASK_FSGID:
                                value = from_kgid(&init_user_ns, current_fsgid());
                                break;
                        case TOMOYO_TASK_PID:
                                value = tomoyo_sys_getpid();
                                break;
                        case TOMOYO_TASK_PPID:
                                value = tomoyo_sys_getppid();
                                break;
                        case TOMOYO_TYPE_IS_SOCKET:
                                value = S_IFSOCK;
                                break;
                        case TOMOYO_TYPE_IS_SYMLINK:
                                value = S_IFLNK;
                                break;
                        case TOMOYO_TYPE_IS_FILE:
                                value = S_IFREG;
                                break;
                        case TOMOYO_TYPE_IS_BLOCK_DEV:
                                value = S_IFBLK;
                                break;
                        case TOMOYO_TYPE_IS_DIRECTORY:
                                value = S_IFDIR;
                                break;
                        case TOMOYO_TYPE_IS_CHAR_DEV:
                                value = S_IFCHR;
                                break;
                        case TOMOYO_TYPE_IS_FIFO:
                                value = S_IFIFO;
                                break;
                        case TOMOYO_MODE_SETUID:
                                value = S_ISUID;
                                break;
                        case TOMOYO_MODE_SETGID:
                                value = S_ISGID;
                                break;
                        case TOMOYO_MODE_STICKY:
                                value = S_ISVTX;
                                break;
                        case TOMOYO_MODE_OWNER_READ:
                                value = 0400;
                                break;
                        case TOMOYO_MODE_OWNER_WRITE:
                                value = 0200;
                                break;
                        case TOMOYO_MODE_OWNER_EXECUTE:
                                value = 0100;
                                break;
                        case TOMOYO_MODE_GROUP_READ:
                                value = 0040;
                                break;
                        case TOMOYO_MODE_GROUP_WRITE:
                                value = 0020;
                                break;
                        case TOMOYO_MODE_GROUP_EXECUTE:
                                value = 0010;
                                break;
                        case TOMOYO_MODE_OTHERS_READ:
                                value = 0004;
                                break;
                        case TOMOYO_MODE_OTHERS_WRITE:
                                value = 0002;
                                break;
                        case TOMOYO_MODE_OTHERS_EXECUTE:
                                value = 0001;
                                break;
                        case TOMOYO_EXEC_ARGC:
                                if (!bprm)
                                        goto out;
                                value = bprm->argc;
                                break;
                        case TOMOYO_EXEC_ENVC:
                                if (!bprm)
                                        goto out;
                                value = bprm->envc;
                                break;
                        case TOMOYO_NUMBER_UNION:
                                /* Fetch values later. */
                                break;
                        default:
                                if (!obj)
                                        goto out;
                                if (!obj->validate_done) {
                                        tomoyo_get_attributes(obj);
                                        obj->validate_done = true;
                                }
                                {
                                        u8 stat_index;
                                        struct tomoyo_mini_stat *stat;

                                        switch (index) {
                                        case TOMOYO_PATH1_UID:
                                        case TOMOYO_PATH1_GID:
                                        case TOMOYO_PATH1_INO:
                                        case TOMOYO_PATH1_MAJOR:
                                        case TOMOYO_PATH1_MINOR:
                                        case TOMOYO_PATH1_TYPE:
                                        case TOMOYO_PATH1_DEV_MAJOR:
                                        case TOMOYO_PATH1_DEV_MINOR:
                                        case TOMOYO_PATH1_PERM:
                                                stat_index = TOMOYO_PATH1;
                                                break;
                                        case TOMOYO_PATH2_UID:
                                        case TOMOYO_PATH2_GID:
                                        case TOMOYO_PATH2_INO:
                                        case TOMOYO_PATH2_MAJOR:
                                        case TOMOYO_PATH2_MINOR:
                                        case TOMOYO_PATH2_TYPE:
                                        case TOMOYO_PATH2_DEV_MAJOR:
                                        case TOMOYO_PATH2_DEV_MINOR:
                                        case TOMOYO_PATH2_PERM:
                                                stat_index = TOMOYO_PATH2;
                                                break;
                                        case TOMOYO_PATH1_PARENT_UID:
                                        case TOMOYO_PATH1_PARENT_GID:
                                        case TOMOYO_PATH1_PARENT_INO:
                                        case TOMOYO_PATH1_PARENT_PERM:
                                                stat_index =
                                                        TOMOYO_PATH1_PARENT;
                                                break;
                                        case TOMOYO_PATH2_PARENT_UID:
                                        case TOMOYO_PATH2_PARENT_GID:
                                        case TOMOYO_PATH2_PARENT_INO:
                                        case TOMOYO_PATH2_PARENT_PERM:
                                                stat_index =
                                                        TOMOYO_PATH2_PARENT;
                                                break;
                                        default:
                                                goto out;
                                        }
                                        if (!obj->stat_valid[stat_index])
                                                goto out;
                                        stat = &obj->stat[stat_index];
                                        switch (index) {
                                        case TOMOYO_PATH1_UID:
                                        case TOMOYO_PATH2_UID:
                                        case TOMOYO_PATH1_PARENT_UID:
                                        case TOMOYO_PATH2_PARENT_UID:
                                                value = from_kuid(&init_user_ns, stat->uid);
                                                break;
                                        case TOMOYO_PATH1_GID:
                                        case TOMOYO_PATH2_GID:
                                        case TOMOYO_PATH1_PARENT_GID:
                                        case TOMOYO_PATH2_PARENT_GID:
                                                value = from_kgid(&init_user_ns, stat->gid);
                                                break;
                                        case TOMOYO_PATH1_INO:
                                        case TOMOYO_PATH2_INO:
                                        case TOMOYO_PATH1_PARENT_INO:
                                        case TOMOYO_PATH2_PARENT_INO:
                                                value = stat->ino;
                                                break;
                                        case TOMOYO_PATH1_MAJOR:
                                        case TOMOYO_PATH2_MAJOR:
                                                value = MAJOR(stat->dev);
                                                break;
                                        case TOMOYO_PATH1_MINOR:
                                        case TOMOYO_PATH2_MINOR:
                                                value = MINOR(stat->dev);
                                                break;
                                        case TOMOYO_PATH1_TYPE:
                                        case TOMOYO_PATH2_TYPE:
                                                value = stat->mode & S_IFMT;
                                                break;
                                        case TOMOYO_PATH1_DEV_MAJOR:
                                        case TOMOYO_PATH2_DEV_MAJOR:
                                                value = MAJOR(stat->rdev);
                                                break;
                                        case TOMOYO_PATH1_DEV_MINOR:
                                        case TOMOYO_PATH2_DEV_MINOR:
                                                value = MINOR(stat->rdev);
                                                break;
                                        case TOMOYO_PATH1_PERM:
                                        case TOMOYO_PATH2_PERM:
                                        case TOMOYO_PATH1_PARENT_PERM:
                                        case TOMOYO_PATH2_PARENT_PERM:
                                                value = stat->mode & S_IALLUGO;
                                                break;
                                        }
                                }
                                break;
                        }
                        max_v[j] = value;
                        min_v[j] = value;
                        switch (index) {
                        case TOMOYO_MODE_SETUID:
                        case TOMOYO_MODE_SETGID:
                        case TOMOYO_MODE_STICKY:
                        case TOMOYO_MODE_OWNER_READ:
                        case TOMOYO_MODE_OWNER_WRITE:
                        case TOMOYO_MODE_OWNER_EXECUTE:
                        case TOMOYO_MODE_GROUP_READ:
                        case TOMOYO_MODE_GROUP_WRITE:
                        case TOMOYO_MODE_GROUP_EXECUTE:
                        case TOMOYO_MODE_OTHERS_READ:
                        case TOMOYO_MODE_OTHERS_WRITE:
                        case TOMOYO_MODE_OTHERS_EXECUTE:
                                is_bitop[j] = true;
                        }
                }
                if (left == TOMOYO_NUMBER_UNION) {
                        /* Fetch values now. */
                        const struct tomoyo_number_union *ptr = numbers_p++;

                        min_v[0] = ptr->values[0];
                        max_v[0] = ptr->values[1];
                }
                if (right == TOMOYO_NUMBER_UNION) {
                        /* Fetch values now. */
                        const struct tomoyo_number_union *ptr = numbers_p++;

                        if (ptr->group) {
                                if (tomoyo_number_matches_group(min_v[0],
                                                                max_v[0],
                                                                ptr->group)
                                    == match)
                                        continue;
                        } else {
                                if ((min_v[0] <= ptr->values[1] &&
                                     max_v[0] >= ptr->values[0]) == match)
                                        continue;
                        }
                        goto out;
                }
                /*
                 * Bit operation is valid only when counterpart value
                 * represents permission.
                 */
                if (is_bitop[0] && is_bitop[1]) {
                        goto out;
                } else if (is_bitop[0]) {
                        switch (right) {
                        case TOMOYO_PATH1_PERM:
                        case TOMOYO_PATH1_PARENT_PERM:
                        case TOMOYO_PATH2_PERM:
                        case TOMOYO_PATH2_PARENT_PERM:
                                if (!(max_v[0] & max_v[1]) == !match)
                                        continue;
                        }
                        goto out;
                } else if (is_bitop[1]) {
                        switch (left) {
                        case TOMOYO_PATH1_PERM:
                        case TOMOYO_PATH1_PARENT_PERM:
                        case TOMOYO_PATH2_PERM:
                        case TOMOYO_PATH2_PARENT_PERM:
                                if (!(max_v[0] & max_v[1]) == !match)
                                        continue;
                        }
                        goto out;
                }
                /* Normal value range comparison. */
                if ((min_v[0] <= max_v[1] && max_v[0] >= min_v[1]) == match)
                        continue;
out:
                return false;
        }
        /* Check argv[] and envp[] now. */
        if (r->ee && (argc || envc))
                return tomoyo_scan_bprm(r->ee, argc, argv, envc, envp);
        return true;
}
















































  582 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_TIMERQUEUE_H
#define _LINUX_TIMERQUEUE_H

#include <linux/rbtree.h>
#include <linux/timerqueue_types.h>

bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node);
bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node);
struct timerqueue_node *timerqueue_iterate_next(struct timerqueue_node *node);

bool timerqueue_linked_add(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node);

/**
 * timerqueue_getnext - Returns the timer with the earliest expiration time
 *
 * @head: head of timerqueue
 *
 * Returns a pointer to the timer node that has the earliest expiration time.
 */
static inline struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head)
{
        struct rb_node *leftmost = rb_first_cached(&head->rb_root);

        return rb_entry_safe(leftmost, struct timerqueue_node, node);
}

static inline void timerqueue_init(struct timerqueue_node *node)
{
        RB_CLEAR_NODE(&node->node);
}

static inline bool timerqueue_node_queued(struct timerqueue_node *node)
{
        return !RB_EMPTY_NODE(&node->node);
}

static inline void timerqueue_init_head(struct timerqueue_head *head)
{
        head->rb_root = RB_ROOT_CACHED;
}

/* Timer queues with linked nodes */

static __always_inline
struct timerqueue_linked_node *timerqueue_linked_first(struct timerqueue_linked_head *head)
{
        return rb_entry_safe(head->rb_root.rb_leftmost, struct timerqueue_linked_node, node);
}

static __always_inline
struct timerqueue_linked_node *timerqueue_linked_next(struct timerqueue_linked_node *node)
{
        return rb_entry_safe(node->node.next, struct timerqueue_linked_node, node);
}

static __always_inline
struct timerqueue_linked_node *timerqueue_linked_prev(struct timerqueue_linked_node *node)
{
        return rb_entry_safe(node->node.prev, struct timerqueue_linked_node, node);
}

static __always_inline
bool timerqueue_linked_del(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node)
{
        return rb_erase_linked(&node->node, &head->rb_root);
}

static __always_inline void timerqueue_linked_init(struct timerqueue_linked_node *node)
{
        RB_CLEAR_LINKED_NODE(&node->node);
}

static __always_inline bool timerqueue_linked_node_queued(struct timerqueue_linked_node *node)
{
        return !RB_EMPTY_LINKED_NODE(&node->node);
}

static __always_inline void timerqueue_linked_init_head(struct timerqueue_linked_head *head)
{
        head->rb_root = RB_ROOT_LINKED;
}

#endif /* _LINUX_TIMERQUEUE_H */













































































































































































    6 

    4 









   13 


















































    6 
























    6 












































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __NET_SCHED_RED_H
#define __NET_SCHED_RED_H

#include <linux/types.h>
#include <linux/bug.h>
#include <net/pkt_sched.h>
#include <net/inet_ecn.h>
#include <net/dsfield.h>
#include <linux/reciprocal_div.h>

/*        Random Early Detection (RED) algorithm.
        =======================================

        Source: Sally Floyd and Van Jacobson, "Random Early Detection Gateways
        for Congestion Avoidance", 1993, IEEE/ACM Transactions on Networking.

        This file codes a "divisionless" version of RED algorithm
        as written down in Fig.17 of the paper.

        Short description.
        ------------------

        When a new packet arrives we calculate the average queue length:

        avg = (1-W)*avg + W*current_queue_len,

        W is the filter time constant (chosen as 2^(-Wlog)), it controls
        the inertia of the algorithm. To allow larger bursts, W should be
        decreased.

        if (avg > th_max) -> packet marked (dropped).
        if (avg < th_min) -> packet passes.
        if (th_min < avg < th_max) we calculate probability:

        Pb = max_P * (avg - th_min)/(th_max-th_min)

        and mark (drop) packet with this probability.
        Pb changes from 0 (at avg==th_min) to max_P (avg==th_max).
        max_P should be small (not 1), usually 0.01..0.02 is good value.

        max_P is chosen as a number, so that max_P/(th_max-th_min)
        is a negative power of two in order arithmetic to contain
        only shifts.


        Parameters, settable by user:
        -----------------------------

        qth_min                - bytes (should be < qth_max/2)
        qth_max                - bytes (should be at least 2*qth_min and less limit)
        Wlog                       - bits (<32) log(1/W).
        Plog                       - bits (<32)

        Plog is related to max_P by formula:

        max_P = (qth_max-qth_min)/2^Plog;

        F.e. if qth_max=128K and qth_min=32K, then Plog=22
        corresponds to max_P=0.02

        Scell_log
        Stab

        Lookup table for log((1-W)^(t/t_ave).


        NOTES:

        Upper bound on W.
        -----------------

        If you want to allow bursts of L packets of size S,
        you should choose W:

        L + 1 - th_min/S < (1-(1-W)^L)/W

        th_min/S = 32         th_min/S = 4

        log(W)        L
        -1        33
        -2        35
        -3        39
        -4        46
        -5        57
        -6        75
        -7        101
        -8        135
        -9        190
        etc.
 */

/*
 * Adaptative RED : An Algorithm for Increasing the Robustness of RED's AQM
 * (Sally FLoyd, Ramakrishna Gummadi, and Scott Shenker) August 2001
 *
 * Every 500 ms:
 *  if (avg > target and max_p <= 0.5)
 *   increase max_p : max_p += alpha;
 *  else if (avg < target and max_p >= 0.01)
 *   decrease max_p : max_p *= beta;
 *
 * target :[qth_min + 0.4*(qth_min - qth_max),
 *          qth_min + 0.6*(qth_min - qth_max)].
 * alpha : min(0.01, max_p / 4)
 * beta : 0.9
 * max_P is a Q0.32 fixed point number (with 32 bits mantissa)
 * max_P between 0.01 and 0.5 (1% - 50%) [ Its no longer a negative power of two ]
 */
#define RED_ONE_PERCENT ((u32)DIV_ROUND_CLOSEST(1ULL<<32, 100))

#define MAX_P_MIN (1 * RED_ONE_PERCENT)
#define MAX_P_MAX (50 * RED_ONE_PERCENT)
#define MAX_P_ALPHA(val) min(MAX_P_MIN, val / 4)

#define RED_STAB_SIZE        256
#define RED_STAB_MASK        (RED_STAB_SIZE - 1)

struct red_stats {
        u32                prob_drop;        /* Early probability drops */
        u32                prob_mark;        /* Early probability marks */
        u32                forced_drop;        /* Forced drops, qavg > max_thresh */
        u32                forced_mark;        /* Forced m